Tensorflow多线程图像加载

Question

所以我有这个玩具示例代码；

import glob
from tqdm import tqdm
import tensorflow as tf

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images

filenameQ = tf.train.string_input_producer(imgPaths)
reader = tf.WholeFileReader()
key, value = reader.read(filenameQ)

img = tf.image.decode_jpeg(value)
init_op = tf.initialize_all_variables()

with tf.Session() as sess:
    sess.run(init_op)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)
    for i in tqdm(range(10000)):
        img.eval().mean()

加载图像并打印每个图像的均值。 我如何对其进行编辑，以便对图像的加载部分进行多线程处理，这是目前我对tf图像脚本的瓶颈。

Answer 1

编辑（2018/3/5）：使用tf.data API现在更容易获得相同的结果。

import glob
from tqdm import tqdm
import tensorflow as tf

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images

dataset = (tf.data.Dataset.from_tensor_slices(imgPaths)
           .map(lambda x: tf.reduce_mean(tf.decode_jpeg(tf.read_file(x))),
                num_parallel_calls=16)
           .prefetch(128))

iterator = dataset.make_one_shot_iterator()
next_mean = iterator.get_next()

with tf.Session() as sess:
    for i in tqdm(range(10000)):
        sess.run(next_mean)

---

正如sygi在其评论中所建议的那样，可以使用tf.train.QueueRunner定义一些在单独线程中运行的操作，并（通常）将值放入TensorFlow队列中。

import glob
from tqdm import tqdm
import tensorflow as tf

imgPaths = glob.glob("/home/msmith/imgs/*/*") # Some images

filenameQ = tf.train.string_input_producer(imgPaths)

# Define a subgraph that takes a filename, reads the file, decodes it, and                                                                                     
# enqueues it.                                                                                                                                                 
filename = filenameQ.dequeue()
image_bytes = tf.read_file(filename)
decoded_image = tf.image.decode_jpeg(image_bytes)
image_queue = tf.FIFOQueue(128, [tf.uint8], None)
enqueue_op = image_queue.enqueue(decoded_image)

# Create a queue runner that will enqueue decoded images into `image_queue`.                                                                                   
NUM_THREADS = 16
queue_runner = tf.train.QueueRunner(
    image_queue,
    [enqueue_op] * NUM_THREADS,  # Each element will be run from a separate thread.                                                                                       
    image_queue.close(),
    image_queue.close(cancel_pending_enqueues=True))

# Ensure that the queue runner threads are started when we call                                                                                               
# `tf.train.start_queue_runners()` below.                                                                                                                      
tf.train.add_queue_runner(queue_runner)

# Dequeue the next image from the queue, for returning to the client.                                                                                          
img = image_queue.dequeue()

init_op = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init_op)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    for i in tqdm(range(10000)):
        img.eval().mean()