TensorFlow : polynomial regression

Question

I am trying to get a non-linear regression from a CSV data which is available at this link: CSV Data

I want to use polynomial regression. the problem is that the result I am getting from TensorFlow is "None". I cannot find the problem. I think there is something wrong with the model or the cost function. can anybody help? any help would be appreciated.

# importing modules
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import csv
import time

# defining the method for gathering data
# date_idx is the column number of date in the .CSV file
def read(filename, date_idx, date_parse, year, bucket=7):

    # the amount of days in the year : 365 days
    days_in_year = 365

    # defining a dictionary for the frequency
    freq = {}
    # here we are calculating hao many buckets each frequency have?
    # buckets = (which is 7, and by that we mean each frequency is 7 days)
    # we are initializing each frequency with zero
    for period in range(0, int(days_in_year / bucket)):
        freq[period] = 0

    # this opens the file in binary mode('rb' : 'r' for read, 'b' is for binary mode)
    with open(filename, 'r') as csvfile:
        csvreader = csv.reader(csvfile)
        next(csvreader)                   # this escapes the first row since it consists of headers only
        for row in csvreader:
            if row[date_idx] == '':        # each row consists of many columns but if the date is
                continue                   # is unavailable there is no need to check the data
            t = time.strptime(row[date_idx], date_parse)    # converting to the input format
            if t.tm_year == year and t.tm_yday < (days_in_year-1):  # we want the data in specific year
                freq[int(t.tm_yday / bucket)] += 1          # finding the frequency
    return freq

# here i call the method to gather data for me
freq = read(r'C:\My Files\Programming\Python\TensorFlow\CallCenter\311_Call_Center_Tracking_Data__Archived_.csv',
            0, '%m/%d/%Y', 2014)

# here we convert our dictionary into 2 arrays or lists in python
x_temp =[]
y_temp =[]
for key, value in freq.items():
    x_temp.append(key)
    y_temp.append(value)

x_data = np.asarray(x_temp)
y_data = np.asarray(y_temp)

# visualizing the data
plt.scatter(x_data,y_data)
plt.show()

# splitting data with ratio into 2 group : training and test
def split_dataset(x_dataset, y_dataset, ratio):
    arr = np.arange(x_dataset.size)
    np.random.shuffle(arr)
    num_train = int(ratio*x_dataset.size)
    x_train = x_dataset[arr[0:num_train]]
    y_train = y_dataset[arr[0:num_train]]
    x_test = x_dataset[arr[num_train:x_dataset.size]]
    y_test = y_dataset[arr[num_train:y_dataset.size]]
    return x_train,y_train,x_test,y_test

x_train, y_train, x_test, y_test = split_dataset(x_data,y_data, ratio=0.7)

# here we create some place holder for input and output of the session
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)

# defining global variables
learning_rate = 0.01
training_epochs = 100
num_coeffs = 5

# adding regularization (for later use)
#reg_lambda = 0.

# defining the coefficients of the polynomial
w = tf.Variable([0.]*num_coeffs, name='parameter')

# defining the model
def model(X,w):
    terms = []
    for i in range(num_coeffs):
        term = tf.multiply(w[i], tf.pow(X, i))
        terms.append(term)
    return tf.add_n(terms)

y_model = model(X,w)

# defining the cost function
cost = tf.reduce_sum(tf.pow(Y-y_model,2))

# defining training method
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

# initilizing all variables
init = tf.global_variables_initializer()

#runing the model
with tf.Session() as sess:
    sess.run(init)
    for epoch in range(training_epochs):
        training_cost = sess.run(train_op, feed_dict={X:x_train, Y:y_train})
        print(training_cost)

        final_cost = sess.run(cost,feed_dict={X: x_test, Y:y_test})
        print('Final cost = {}'.format(training_cost))

Answer 1

The problem is that training_cost = sess.run(train_op, feed_dict={X:x_train, Y:y_train}) doesn't return the training cost, because train_op is the operation that updates the parameters using gradient descent, not the operation that computes the cost function.

If you want to get the training cost, you should do the following:

_, training_cost = sess.run([train_op, cost], feed_dict={X:x_train, Y:y_train})

Where cost is the operation that you have defined previously as cost = tf.reduce_sum(tf.pow(Y-y_model,2))

Answer 2

I changed the code as follow:

# importing modules
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import csv
import time

# defining the method for gathering data
# date_idx is the column number of date in the .CSV file
from pylint.checkers.raw_metrics import get_type


def read(filename, date_idx, date_parse, year, bucket=7):

    # the amount of days in the year : 365 days
    days_in_year = 365

    # defining a dictionary for the frequency
    freq = {}
    # here we are calculating hao many buckets each frequency have?
    # buckets = (which is 7, and by that we mean each frequency is 7 days)
    # we are initializing each frequency with zero
    for period in range(0, int(days_in_year / bucket)):
        freq[period] = 0

    # this opens the file in binary mode('rb' : 'r' for read, 'b' is for binary mode)
    with open(filename, 'r') as csvfile:
        csvreader = csv.reader(csvfile)
        next(csvreader)                   # this escapes the first row since it consists of headers only
        for row in csvreader:
            if row[date_idx] == '':        # each row consists of many columns but if the date is
                continue                   # is unavailable there is no need to check the data
            t = time.strptime(row[date_idx], date_parse)    # converting to the input format
            if t.tm_year == year and t.tm_yday < (days_in_year-1):  # we want the data in specific year
                freq[int(t.tm_yday / bucket)] += 1          # finding the frequency
    return freq

# here i call the method to gather data for me
freq = read(r'C:\My Files\Programming\Python\TensorFlow\CallCenter\311_Call_Center_Tracking_Data__Archived_.csv',
            0, '%m/%d/%Y', 2014)

# here we convert our dictionary into 2 arrays or lists in python
x_temp =[]
y_temp =[]
for key, value in freq.items():
    x_temp.append(key)
    y_temp.append(value)

x_data = np.asarray(x_temp)
x_data = x_data.astype(float)
y_data = np.asarray(y_temp)
y_data = y_data.astype(float)

print(x_data)
print(y_data)

# visualizing the data
plt.scatter(x_data,y_data)
plt.show()

# splitting data with ratio into 2 group : training and test
def split_dataset(x_dataset, y_dataset, ratio):
    arr = np.arange(x_dataset.size)
    np.random.shuffle(arr)
    num_train = int(ratio*x_dataset.size)
    x_train = x_dataset[arr[0:num_train]]
    y_train = y_dataset[arr[0:num_train]]
    x_test = x_dataset[arr[num_train:x_dataset.size]]
    y_test = y_dataset[arr[num_train:y_dataset.size]]
    return x_train,y_train,x_test,y_test

x_train, y_train, x_test, y_test = split_dataset(x_data,y_data, ratio=0.7)
print(type(x_train[0]))
# here we create some place holder for input and output of the session
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)

# defining global variables
learning_rate = 0.01
training_epochs = 100
num_coeffs = 5

# adding regularization (for later use)
#reg_lambda = 0.

# defining the coefficients of the polynomial
w = tf.Variable([0.]*num_coeffs, name='parameter')

# defining the model
def model(X,w):
    terms = []
    for i in range(num_coeffs):
        term = tf.multiply(w[i], tf.pow(X, i))
        terms.append(term)
    return tf.add_n(terms)

y_model = model(X,w)

# defining the cost function
cost = tf.reduce_sum(tf.pow(Y-y_model,2))

# defining training method
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

# initilizing all variables
init = tf.global_variables_initializer()

#runing the model
with tf.Session() as sess:
    sess.run(init)
    for epoch in range(training_epochs):
        _, training_cost = sess.run([train_op, cost], feed_dict={X: x_train, Y: y_train})
        print('Training_cost = {}'.format(training_cost))

        final_cost = sess.run(cost,feed_dict={X: x_test, Y:y_test})
        print('Final cost = {}'.format(training_cost))

The result changed from "nan" to this:

Training_cost = 11020688384.0
Final cost = 11020688384.0
Training_cost = 9.952021814670212e+34
Final cost = 9.952021814670212e+34
Training_cost = inf
Final cost = inf
Training_cost = inf
Final cost = inf
Training_cost = inf
Final cost = inf
Training_cost = nan
Final cost = nan
Training_cost = nan
Final cost = nan

I just made everything into a float, since multiply just accepts 2 floats.

Answer 3

I just changed the code as follows. the code is working now but the result is off it still needs more optimization I think on model definition. thanks to @gcucurull I was able to pull it off.

# importing modules
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import csv
import time

# defining the method for gathering data
# date_idx is the column number of date in the .CSV file
from pylint.checkers.raw_metrics import get_type


def read(filename, date_idx, date_parse, year, bucket=7):

    # the amount of days in the year : 365 days
    days_in_year = 365

    # defining a dictionary for the frequency
    freq = {}
    # here we are calculating hao many buckets each frequency have?
    # buckets = (which is 7, and by that we mean each frequency is 7 days)
    # we are initializing each frequency with zero
    for period in range(0, int(days_in_year / bucket)):
        freq[period] = 0

    # this opens the file in binary mode('rb' : 'r' for read, 'b' is for binary mode)
    with open(filename, 'r') as csvfile:
        csvreader = csv.reader(csvfile)
        next(csvreader)                   # this escapes the first row since it consists of headers only
        for row in csvreader:
            if row[date_idx] == '':        # each row consists of many columns but if the date is
                continue                   # is unavailable there is no need to check the data
            t = time.strptime(row[date_idx], date_parse)    # converting to the input format
            if t.tm_year == year and t.tm_yday < (days_in_year-1):  # we want the data in specific year
                freq[int(t.tm_yday / bucket)] += 1          # finding the frequency
    return freq

# here i call the method to gather data for me
freq = read(r'C:\My Files\Programming\Python\TensorFlow\CallCenter\311_Call_Center_Tracking_Data__Archived_.csv',
            0, '%m/%d/%Y', 2014)

# here we convert our dictionary into 2 arrays or lists in python
x_temp =[]
y_temp =[]
for key, value in freq.items():
    x_temp.append(key)
    y_temp.append(value)

x_data = np.asarray(x_temp)
x_data = x_data.astype(float)
y_data = np.asarray(y_temp)
y_data = y_data.astype(float)

print(x_data)
print(y_data)

# visualizing the data
#plt.scatter(x_data,y_data)
#plt.show()

# splitting data with ratio into 2 group : training and test
def split_dataset(x_dataset, y_dataset, ratio):
    arr = np.arange(x_dataset.size)
    np.random.shuffle(arr)
    num_train = int(ratio*x_dataset.size)
    x_train = x_dataset[arr[0:num_train]]
    y_train = y_dataset[arr[0:num_train]]
    x_test = x_dataset[arr[num_train:x_dataset.size]]
    y_test = y_dataset[arr[num_train:y_dataset.size]]
    return x_train,y_train,x_test,y_test

x_train, y_train, x_test, y_test = split_dataset(x_data,y_data, ratio=0.7)
print(type(x_train[0]))

print(x_train)

# defining global variables
learning_rate = 0.000001
training_epochs = 10000
num_coeffs = 5

# defining the coefficients of the polynomial
w = tf.Variable(
    tf.truncated_normal([num_coeffs,1], mean=0.0,stddev= 1.0, dtype=tf.float64))

# adding bias
b = tf.Variable(tf.zeros(1,dtype=tf.float64))

# predefining the model
def model(x, y):
    # this predicts the y based on the given weight
    temp = []
    for i in range(num_coeffs):
        temp.append(tf.add(w[i],tf.pow(x,i)))
    prediction = tf.add(tf.reduce_sum(temp),b)
    # this is the cost function
    errors =tf.square(y - prediction)
    return [prediction, errors]

# defining the model
y, cost = model(x_train, y_train)

# defining training method
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

# initializing all variables
init = tf.global_variables_initializer()

#runing the model
with tf.Session() as sess:
    sess.run(init)
    for epoch in list(range(training_epochs)):
        sess.run(optimizer)
        if epoch%1000 ==0:
            print('Training cost = \n',sess.run(cost))
    print('---------------------------------------------------------------------------------')
    print('---------------------------------------------------------------------------------')
    y_prediction, cost_prediction = model(x_test, y_test)
    print(sess.run(y_prediction))

print(y_test[-1])