如何使用单个 GPU 在 tensorflow python 中同时运行多个模型？

Question

I am trying to use my laptop as server for faceRecognition and speechSynthesis for my project on raspberry pi. So I created a program which loads both the models initially and then wait for the request to come. But when I start the program, initially faceRecognition model loads successfully but then at the time of loading speechSynthesis model, it gives me as error regarding the tf.saver . code:

Server-

def findFaceMatch():
    image_file = request.files.get("imagefile")
    image_file.save("image.jpg")
    print("sent for check")
    response = face_match_demo.recognizeFace(os.path.join(os.getcwd(),"image.jpg"))
    return response, 200 

@api.route("/synthesize/<string:text>") 
def synthesizeVoice(text):
    print(text)
    with open("F:/file.wav", 'wb') as f:
        f.write(synthesizer.synthesize(text))
    return send_from_directory("F:/","file.wav", as_attachment=True), 200

Face Recognition-

import tensorflow as tf
import numpy as np
from . import facenet
from .align import detect_face
import cv2
import imutils
import os
import pickle
import time


minsize = 20
threshold = [0.6, 0.7, 0.7]
factor = 0.709
margin = 44
input_image_size = 160

def load_models(session):
    global sess
    sess = session
    global pnet, rnet, onet
    pnet, rnet, onet = detect_face.create_mtcnn(sess, os.path.join(os.getcwd(),"Face_recognition","align"))

    facenet.load_model(os.path.join(os.getcwd(),"Face_recognition","20170512-110547\\20170512-110547.pb"))

    global images_placeholder
    images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
    global embeddings
    embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
    global phase_train_placeholder
    phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
    global embedding_size
    embedding_size = embeddings.get_shape()[1]

def getFace(img):
    faces = []
    img_size = np.asarray(img.shape)[0:2]
    bounding_boxes, _ = detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor)
    if not len(bounding_boxes) == 0:
        for face in bounding_boxes:
            if face[4] > 0.50:
                det = np.squeeze(face[0:4])
                bb = np.zeros(4, dtype=np.int32)
                bb[0] = np.maximum(det[0] - margin / 2, 0)
                bb[1] = np.maximum(det[1] - margin / 2, 0)
                bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
                bb[3] = np.minimum(det[3] + margin / 2, img_size[0])
                cropped = img[bb[1]:bb[3], bb[0]:bb[2], :]
                resized = cv2.resize(cropped, (input_image_size,input_image_size),interpolation=cv2.INTER_CUBIC)
                prewhitened = facenet.prewhiten(resized)
                faces.append(getEmbedding(prewhitened))
    return faces

def getEmbedding(resized):
    reshaped = resized.reshape(-1,input_image_size,input_image_size,3)
    feed_dict = {images_placeholder: reshaped, phase_train_placeholder: False}
    embedding = sess.run(embeddings, feed_dict=feed_dict)
    return embedding

def compare2face(img1):
    print("checking")
    face2 = getFace(img1)
    face1 = []
    with open(os.path.join(os.getcwd(),"Face_recognition","trained_knn_model.PB"), 'rb') as f:
        for i in range(4):
            face1.append(pickle.load(f))
    names = ["x","y","z","p"]
    print("verifying name")
    for i in range(0,len(face1)):
        if face1[i] and face2:
            # calculate Euclidean distance
            dist = np.sqrt(np.sum(np.square(np.subtract(face1[i], face2[0]))))
            if dist <= 0.8:
                return "dist: "+str(dist)+"\nhello "+names[i]
    return  "Person not found"

def recognizeFace(image_path):
    image = cv2.imread(image_path)
    response = compare2face(image)
    return response

Speech Synthesis:

import io
import numpy as np
import tensorflow as tf
from .hparams import hparams
from librosa import effects
from .models import create_model
from .text import text_to_sequence
from .util import audio


class Synthesizer:
  def load(self, checkpoint_path, sess,  model_name='tacotron'):
    print('Constructing model: %s' % model_name)
    inputs = tf.placeholder(tf.int32, [1, None], 'inputs')
    input_lengths = tf.placeholder(tf.int32, [1], 'input_lengths')
    with tf.variable_scope('model') as scope:
      self.model = create_model(model_name, hparams)
      self.model.initialize(inputs, input_lengths)
      self.wav_output = audio.inv_spectrogram_tensorflow(self.model.linear_outputs[0])

    print('Loading checkpoint: %s' % checkpoint_path)
    # self.session = tf.Session()
    self.session = sess
    self.session.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    saver.restore(self.session, checkpoint_path)


  def synthesize(self, text):
    cleaner_names = [x.strip() for x in hparams.cleaners.split(',')]
    seq = text_to_sequence(text, cleaner_names)
    feed_dict = {
      self.model.inputs: [np.asarray(seq, dtype=np.int32)],
      self.model.input_lengths: np.asarray([len(seq)], dtype=np.int32)
    }
    wav = self.session.run(self.wav_output, feed_dict=feed_dict)
    wav = audio.inv_preemphasis(wav)
    wav = wav[:audio.find_endpoint(wav)]
    out = io.BytesIO()
    audio.save_wav(wav, out)
    return out.getvalue()

error is as follows:

2019-11-11 21:48:04.408636: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at save_restore_v2_ops.cc:184 : Not found: Key onet/conv1/biases not found in checkpoint
Traceback (most recent call last):
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1356, in _do_call
    return fn(*args)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1341, in _run_fn
    options, feed_dict, fetch_list, target_list, run_metadata)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
    run_metadata)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
  (0) Not found: Key onet/conv1/biases not found in checkpoint
         [[{{node save/RestoreV2}}]]
         [[save/RestoreV2/_617]]
  (1) Not found: Key onet/conv1/biases not found in checkpoint
         [[{{node save/RestoreV2}}]]
0 successful operations.
0 derived errors ignored.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1286, in restore
    {self.saver_def.filename_tensor_name: save_path})
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 950, in run
    run_metadata_ptr)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1173, in _run
    feed_dict_tensor, options, run_metadata)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1350, in _do_run
    run_metadata)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1370, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
  (0) Not found: Key onet/conv1/biases not found in checkpoint
         [[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
         [[save/RestoreV2/_617]]
  (1) Not found: Key onet/conv1/biases not found in checkpoint
         [[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'save/RestoreV2':
  File "commonServer.py", line 38, in <module>
    synthesizer.load(model_path,sess)
  File "F:\Backend\Text_To_Speech\synthesizer.py", line 25, in load
    saver = tf.train.Saver()
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
    self.build()
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
    self._build(self._filename, build_save=True, build_restore=True)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
    build_restore=build_restore)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
    restore_sequentially, reshape)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
    restore_sequentially)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
    return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1696, in restore_v2
    name=name)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
    return func(*args, **kwargs)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
    op_def=op_def)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
    self._traceback = tf_stack.extract_stack()


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1296, in restore
    names_to_keys = object_graph_key_mapping(save_path)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1614, in object_graph_key_mapping
    object_graph_string = reader.get_tensor(trackable.OBJECT_GRAPH_PROTO_KEY)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\pywrap_tensorflow_internal.py", line 678, in get_tensor
    return CheckpointReader_GetTensor(self, compat.as_bytes(tensor_str))
tensorflow.python.framework.errors_impl.NotFoundError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "commonServer.py", line 38, in <module>
    synthesizer.load(model_path,sess)
  File "F:\Backend\Text_To_Speech\synthesizer.py", line 26, in load
    saver.restore(self.session, checkpoint_path)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1302, in restore
    err, "a Variable name or other graph key that is missing")
tensorflow.python.framework.errors_impl.NotFoundError: Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:

2 root error(s) found.
  (0) Not found: Key onet/conv1/biases not found in checkpoint
         [[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
         [[save/RestoreV2/_617]]
  (1) Not found: Key onet/conv1/biases not found in checkpoint
         [[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'save/RestoreV2':
  File "commonServer.py", line 38, in <module>
    synthesizer.load(model_path,sess)
  File "F:\Backend\Text_To_Speech\synthesizer.py", line 25, in load
    saver = tf.train.Saver()
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
    self.build()
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
    self._build(self._filename, build_save=True, build_restore=True)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
    build_restore=build_restore)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
    restore_sequentially, reshape)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
    restore_sequentially)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
    return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1696, in restore_v2
    name=name)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
    return func(*args, **kwargs)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
    op_def=op_def)
  File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
    self._traceback = tf_stack.extract_stack()

Answer 1

Initially I was trying to load face_recognition model first and then loaded the TTS model. Out of nothing I tried loading TTS model first and then Face_recognition model, and guess what? It worked perfectly without any errors. I don't know how it worked in reverse way. If anyone still knows answer to this, please help.

Answer 2

At least up to TF 2.0 (not sure about the version you have), loading multiple models into one graph could cause problems: See load multiple models in Tensorflow

I hope this helps.