How to use tensor flow to build a CNN model where each “branch” is separate until being joined at the last step?

Question

This is quite hard to express in words clearly, so please let me know if you have any questions.

I am training a image recognition model using CNN. But I have images of different angles, so I want to do several layers with each angle of n pictures. In this process, I decrease the output of each angle to 1, and in the end, I combine this output of all angles and do a 2 layers neural network.

I tried to train one angle as test and got working result, then I put each layer of each angle into a list as I go along the same step,(a list of convolutional layer1 for all angles, a list of pooling layer1, a list of convolutional layer2, a list of pooling layer2 ... ) except I add the last step of combining them and do the final logistic layer and get a 0/1 output.

I got tons of errors like Key conv2d_7/bias not found in checkpoint.

So instead of debugging, I'm wondering if this is a valid way to do it in the first place? Is it possible that when I put all the layers in different lists, the model can't find some of the values in the flow?

If not, what should I do to achieve this goal of training each angle separately and then combine the result and do final training towards 0/1?

Thanks a lot.

Here is the code:

def model_train(features, labels, mode):
    input_list = []
    conv1_list = []
    pool1_list = []
    conv2_list = []
    pool2_list = []
    conv3_list = []
    pool3_list = []
    pool3_flat_list = []
    dense1_list = []
    dense2_list = []
    logit1_list = []
    feature = tf.transpose(features["x"],[1,0,2,3])
    for i in range(0,11,1):
        input_list.append(tf.reshape(feature[i], [-1, 73, 135, 1]))

        conv1_list.append(tf.layers.conv2d(
            inputs=input_list[i],
            filters=10,
            kernel_size=[5, 5],
            padding="same",
            activation=tf.nn.relu))
        pool1_list.append(tf.layers.max_pooling2d(inputs=conv1_list[i], pool_size=[2, 2], strides=2))
        conv2_list.append(tf.layers.conv2d(
            inputs=pool1_list[i],
            filters=20,
            kernel_size=[3, 3],
            padding="same",
            activation=tf.nn.relu))
        pool2_list.append(tf.layers.max_pooling2d(inputs=conv2_list[i], pool_size=[2, 2], strides=2))
        conv3_list.append(tf.layers.conv2d(
            inputs=pool2_list[i],
            filters=30,
            kernel_size=[5, 5],
            padding="same",
            activation=tf.nn.relu))
        pool3_list.append(tf.layers.max_pooling2d(inputs=conv3_list[i], pool_size=[3, 3], strides=3))
        pool3_flat_list.append(tf.reshape(pool3_list[i], [-1, 6*11*30]))
        dense1_list.append(tf.layers.dense(inputs=pool3_flat_list[i], units=512, activation=tf.nn.relu))
        dense2_list.append(tf.layers.dense(inputs=dense1_list[i], units=16, activation=tf.nn.relu))
        logit1_list.append(tf.layers.dense(inputs=dense2_list[i],units=1))


    # nn of 11 separate results
    input2 = tf.reshape(logit1_list,[11,-1])
    input2 = tf.transpose(input2)
    dense3 = tf.layers.dense(inputs = input2, units = 64)
    logit2 = tf.layers.dense(inputs = dense3, units = 2)



    predictions = {
        # Generate predictions (for PREDICT and EVAL mode)
        "classes": tf.argmax(input=logit2, axis=1),
        # Add `softmax_tensor` to the graph. It is used for PREDICT and by the
        # `logging_hook`.
        "probabilities": tf.nn.softmax(logit2, name="softmax_tensor")
    }

    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

    onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=2)
    onehot_labels = tf.squeeze(onehot_labels, axis=1)
    loss = tf.losses.softmax_cross_entropy(
        onehot_labels=onehot_labels, logits=logit2)

    # Configure the Training Op (for TRAIN mode)
    if mode == tf.estimator.ModeKeys.TRAIN:
        optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
        train_op = optimizer.minimize(
            loss=loss,
            global_step=tf.train.get_global_step())
        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)

    # Add evaluation metrics (for EVAL mode)
    eval_metric_ops = {
        "accuracy": tf.metrics.accuracy(
            labels=labels, predictions=predictions["classes"])}
    return tf.estimator.EstimatorSpec(
        mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)


def main(argv=None):
    dropbox_dir = '/Users/someone/Dropbox'
    csv_dir = dropbox_dir + '/ML/CSInde/stage1_labels.csv'
    zone13_dir = dropbox_dir + '/ML/zone13/'

    image_list, label_list = read_labeled_image_list(csv_dir)
    labels = np.asarray(label_list, dtype=np.int32)
    labels = np.reshape(labels, [1147, 1])

    images = read_images_from_disk(image_list)

    model = tf.estimator.Estimator(
        model_fn=model_train, model_dir="/tmp/test_model")
    tensors_to_log = {"probabilities": "softmax_tensor"}

    logging = tf.train.LoggingTensorHook(
        tensors=tensors_to_log, every_n_iter=50)

    train_input_fn = tf.estimator.inputs.numpy_input_fn(
        x={"x": images},
        y=labels,
        batch_size=100,
        num_epochs=None,
        shuffle=False)
    model.train(
        input_fn=train_input_fn,
        steps=20000,
        hooks=[logging]
    )
    print('finishTrain++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
    # a = tf.Print(tf.shape(images), [tf.shape(images)])
    # b = tf.add(a, a).eval()
    eval_input_fn = tf.estimator.inputs.numpy_input_fn(
        x={"x": images},
        y=labels,
        num_epochs=1,
        shuffle=False)
    eval_results = model.evaluate(input_fn=eval_input_fn)
    print(eval_results)

I didn't post the part where I read in data since I don't think that is causing the problems. So I just post the model part and the main method.

Thanks.

Answer 1

Yes, what you did should work. The code is not attached so I cannot see what is the exact problem in saving/loading of the model.

My guess though, is that you don't follow the training code convention in the evaluation code. The reason I think so is because the name conv2d_7/bias is not angle dependent.

So - if you declared your layers with angle specific names, or prefix, make sure you also use that same name in your evaluation code. ie, for angle 30, you have to define conv2d_7-angle_30/bias (or however you named it), instead of conv2d_7/bias .