Tensorflow incompatible shapes between batch size and actual output

Question

I'm currently trying to create a simple web application for interactive neural network experimentation. I'm quite new to Tensorflow and machine learning in general so I wanted to start with a simple time-series regression on S&P500.

The problem I'm having is with the following error:

InvalidArgumentError (see above for traceback): Incompatible shapes: [32,1] vs. [1248,1]

In the case the batch size is 32 and the actual data size is 1248. It arises from the following line of code when the session is run:

tMSE = tf.reduce_mean(tf.square(y_hat - train_y))

Here's the source code

def retrieve_data():
"""Retrieves the data - to be expanded for custom database access + S3 retrieval + URL"""
result = pd.read_csv('snp_data.csv', parse_dates=['Date'], index_col=['Date'])
return result

def get_features(data, columns):
    features = data.ix[:, columns]
    return features

def preprocess(data):
    """Data preprocessing"""
    result = (data - data.mean()) / data.std(ddof=0)
    result = result.fillna(0)
    return result

def init_weights(shape):
    """ Weights initialization """
    weights = tf.random_normal(shape=shape, stddev=0.1)
    return tf.Variable(weights)

def forwardprop(X, w_1, w_2):
    """Forward propagation"""
    h = tf.nn.relu(tf.matmul(X, w_1))
    y_hat = tf.matmul(h, w_2)
    return y_hat

@app.route('/train')
def train():
    data = retrieve_data()

    train_x = get_features(data, columns=['Open', 'Close'])
    train_x = preprocess(data=train_x).as_matrix().astype(np.float32)
    train_x = train_x[:(len(train_x) - (len(train_x) % 32))]

    train_y = get_features(data, columns=['Adj Close']).as_matrix().astype(np.float32)
    train_y = train_y[:(len(train_y) - (len(train_y) % 32))]

    # Number of input nodes
    n_features = train_x.shape[1]

    # Number of output nodes
    output_nodes = train_y.shape[1]

    # Number of hidden nodes
    hidden_nodes = 20

    # TF Placeholders for the inputs and outputs
    tx = tf.placeholder(tf.float32, shape=(None, n_features))
    ty = tf.placeholder(tf.float32, shape=(None, output_nodes))

    # Weight initializations
    tW1 = init_weights(shape=(n_features, hidden_nodes))
    tW2 = init_weights(shape=(hidden_nodes, output_nodes))

    # Forward propagation
    y_hat = forwardprop(tx, tW1, tW2)

    # Backward Propagation
    tMSE = tf.reduce_mean(tf.square(y_hat - train_y))
    learning_rate = 0.025
    tOptimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
    tOptimize = tOptimizer.minimize(tMSE)

    batch_size = 32
    n_epochs = 8

    init = tf.global_variables_initializer()

    with tf.Session() as sess:
        sess.run(init)
        for i_e in range(n_epochs):
            for i in range(0, train_x.shape[0], batch_size):
                batch_X = train_x[i:i + batch_size, ...]
                batch_y = train_y[i:i + batch_size]

                _, loss = sess.run([tOptimize, tMSE], feed_dict={tx: batch_X, ty: batch_y})
                print(i, loss)
    return 'Flask Dockerized'

And here's the logged error:

Traceback (most recent call last):
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1997, in __call__
    return self.wsgi_app(environ, start_response)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1985, in wsgi_app
    response = self.handle_exception(e)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1540, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1982, in wsgi_app
    response = self.full_dispatch_request()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1614, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1517, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1612, in full_dispatch_request
    rv = self.dispatch_request()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1598, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/{PROJECT_PATH}/web/app.py", line 85, in train
    _, loss = sess.run([tOptimize, tMSE], feed_dict={tx: batch_X, ty: batch_y})
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 895, in run
    run_metadata_ptr)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1124, in _run
    feed_dict_tensor, options, run_metadata)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1321, in _do_run
    options, run_metadata)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1340, in _do_call
    raise type(e)(node_def, op, message)
InvalidArgumentError: Incompatible shapes: [32,1] vs. [1248,1]
     [[Node: sub = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](MatMul_1, sub/y)]]

Caused by op u'sub', defined at:
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 783, in __bootstrap
    self.__bootstrap_inner()
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 702, in inner
    srv.serve_forever()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 539, in serve_forever
    HTTPServer.serve_forever(self)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 238, in serve_forever
    self._handle_request_noblock()
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 295, in _handle_request_noblock
    self.process_request(request, client_address)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 321, in process_request
    self.finish_request(request, client_address)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 334, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 655, in __init__
    self.handle()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 232, in handle
    rv = BaseHTTPRequestHandler.handle(self)
  File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/BaseHTTPServer.py", line 340, in handle
    self.handle_one_request()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 267, in handle_one_request
    return self.run_wsgi()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 209, in run_wsgi
    execute(self.server.app)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/serving.py", line 199, in execute
    for data in application_iter:
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/werkzeug/debug/__init__.py", line 284, in debug_application
    app_iter = self.app(environ, start_response)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1997, in __call__
    return self.wsgi_app(environ, start_response)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1982, in wsgi_app
    response = self.full_dispatch_request()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1612, in full_dispatch_request
    rv = self.dispatch_request()
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/flask/app.py", line 1598, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/{PROJECT_PATH}/web/app.py", line 68, in train
    tMSE = tf.reduce_mean(tf.square(y_hat - train_y))
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 865, in binary_op_wrapper
    return func(x, y, name=name)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 2629, in _sub
    result = _op_def_lib.apply_op("Sub", x=x, y=y, name=name)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
    op_def=op_def)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2630, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/{PROJECT_PATH}/venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1204, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): Incompatible shapes: [32,1] vs. [1248,1]
     [[Node: sub = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](MatMul_1, sub/y)]]

Answer 1

You should change your tMSE code:

# original wrong code: tMSE = tf.reduce_mean(tf.square(y_hat - train_y))
tMSE = tf.reduce_mean(tf.square(y_hat - ty))