结合两种不同的深度学习模型进行评估

Question

I have 2 different Deep Learning Model for defect detection .One is the Object detection model and other one is Semantic segmentation model. The aim is to integrate both into single prediction algorithm. I am looking for the best way to combine these two models to be able to predict the outcome. I mainly want to evaluate both the models in combined form and calculate mean average precision(mAP) Is there any way I can do it. I am not able to evaluate as I am stuck during the prediction part of the code.

def _get_detections(generator, model, unetmodel=None, score_threshold=0.05, max_detections=100, save_path=None):
    """ Get the detections from the model using the generator.

    The result is a list of lists such that the size is:
        all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes]

    # Arguments
        generator       : The generator used to run images through the model.
        model           : The model to run on the images.
        score_threshold : The score confidence threshold to use.
        max_detections  : The maximum number of detections to use per image.
        save_path       : The path to save the images with visualized detections to.
    # Returns
        A list of lists containing the detections for each image in the generator.
    """
    all_detections = [[None for i in range(generator.num_classes()) if generator.has_label(i)] for j in range(generator.size())]
    all_inferences = [None for i in range(generator.size())]

    for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '):
        test_class = generator.load_annotations(i)['test_class']
        print("------")
        if test_class == 'scratches':
            usemodel = unetmodel
            print("Using unetmodel")
            raw_image    = generator.load_image(i)
            image, scale = generator.resize_image(raw_image.copy())
            image = generator.preprocess_image(image)

            if keras.backend.image_data_format() == 'channels_first':
                image = image.transpose((2, 0, 1))

            # run network
            start = time.time()
            #print (unetmodel.predict_on_batch(np.expand_dims(image, axis=0))[:3])
            #prediction = unetmodel.predict_on_batch(np.expand_dims(image, axis=0))[:3]
            boxes, scores, labels = (unetmodel.predict_on_batch(np.expand_dims(image, axis=0))[0,:,:,0] > 0.4).astype(np.uint8)
            #boxes, scores, labels = unetmodel.predict_on_batch(np.expand_dims(image, axis=0))[:3]
            inference_time = time.time() - start

            # correct boxes for image scale
            boxes /= scale

            # select indices which have a score above the threshold
            indices = np.where(scores[0, :] > score_threshold)[0]

            # select those scores
            scores = scores[0][indices]

            # find the order with which to sort the scores
            scores_sort = np.argsort(-scores)[:max_detections]

            # select detections
            image_boxes      = boxes[0, indices[scores_sort], :]
            image_scores     = scores[scores_sort]
            image_labels     = labels[0, indices[scores_sort]]
            image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1)    

        else:
            usemodel = model
            print("Using retinanetmodel")
        
            raw_image    = generator.load_image(i)
            image, scale = generator.resize_image(raw_image.copy())
            image = generator.preprocess_image(image)

            if keras.backend.image_data_format() == 'channels_first':
                image = image.transpose((2, 0, 1))

            # run network
            start = time.time()
            boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))[:3]
            inference_time = time.time() - start

            # correct boxes for image scale
            boxes /= scale

            # select indices which have a score above the threshold
            indices = np.where(scores[0, :] > score_threshold)[0]

            # select those scores
            scores = scores[0][indices]

            # find the order with which to sort the scores
            scores_sort = np.argsort(-scores)[:max_detections]

            # select detections
            image_boxes      = boxes[0, indices[scores_sort], :]
            image_scores     = scores[scores_sort]
            image_labels     = labels[0, indices[scores_sort]]
            image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1)

        if save_path is not None:
            draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name)
            draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name, score_threshold=score_threshold)

            cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image)

        # copy detections to all_detections
        for label in range(generator.num_classes()):
            if not generator.has_label(label):
                continue

            #all_detections[i][label] = np.concatenate([image_detections_unet[image_detections_unet[:, -1] == label, :-1]], [image_detections_rnet[image_detections_rnet[:, -1] == label, :-1]])
            all_detections[i][label] = image_detections[image_detections[:, -1] == label, :-1]

        #all_inferences[i] = np.concatenate(inference_time1, inference_time2)
        all_inferences[i] = inference_time   

    return all_detections, all_inferences

I have done some modification. please suggest any changes. As I am getting the below error.

WARNING:tensorflow:Model was constructed with shape (None, 256, 256, 3) for input Tensor("input_4:0", shape=(None, 256, 256, 3), dtype=float32), but it was called on an input with incompatible shape (1, 800, 800, 3).
Traceback (most recent call last):
  File "C:\Users\user\venv1\keras-retinanet\keras_retinanet\bin\evaluate_new.py", line 200, in <module>
    main()
  File "C:\Users\user\venv1\keras-retinanet\keras_retinanet\bin\evaluate_new.py", line 177, in main
    save_path=args.save_path
  File "C:\Users\user\venv1\keras-retinanet\keras_retinanet\bin\..\..\keras_retinanet\utils\eval_new1.py", line 219, in evaluate
    all_detections, all_inferences = _get_detections(generator, model, unetmodel, score_threshold=score_threshold, max_detections=max_detections, save_path=save_path)
  File "C:\Users\user\venv1\keras-retinanet\keras_retinanet\bin\..\..\keras_retinanet\utils\eval_new1.py", line 94, in _get_detections
    boxes, scores, labels = (unetmodel.predict_on_batch(np.expand_dims(image, axis=0))[0,:,:,0] > 0.4).astype(np.uint8)
ValueError: too many values to unpack (expected 3)

Any help is appreciated. Thank you in advance.

Answer 1

You can use tf.keras.layers.Concatenate from Tensorflow Keras API to combine two models

# Concatenate last layer of model1 and model2
concat = tf.keras.layers.Concatenate()([dense_1, dense_2])

n_classes = 10
# output layer
output = tf.keras.layers.Dense(units=n_classes,
                               activation=tf.keras.activations.softmax)(concat)

Complete _model = tf.keras.Model(inputs=[input_1, input_2], outputs=[output])

complete _model.summary()