Python line_profiler with multithread

Question

I would like to profile a code using line_profiler, where there are threads. With a native/naive code implementation, the following code will not capture the time spent by the thread in do_other-stuff

from line_profiler import LineProfiler
import random, threading, time

def do_other_stuff(numbers):
    time.sleep(1)
    s = sum(numbers)

def do_stuff(numbers):
    my_thread = threading.Thread(target=do_other_stuff,args=[numbers])
    my_thread.start()

numbers = [random.randint(1,100) for i in range(1000)]
lp = LineProfiler()
lp.add_function(do_other_stuff)
lp_wrapper = lp(do_stuff)
lp_wrapper(numbers)
lp.print_stats()

The results being

Timer unit: 1e-07 s

Total time: 0 s
File: <ipython-input-5-304263bfd927>
Function: do_other_stuff at line 4

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
     4                                           def do_other_stuff(numbers):
     5                                               time.sleep(1)
     6                                               s = sum(numbers)

Total time: 0.0008575 s
File: <ipython-input-5-304263bfd927>
Function: do_stuff at line 8

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
     8                                           def do_stuff(numbers):
     9         1        578.0    578.0      6.7      my_thread = threading.Thread(target=do_other_stuff,args=[numbers])
    10         1       7997.0   7997.0     93.3      my_thread.start()

Any idea how to include the thread in the line profile?

Answer 1

You can use a profiler per thread.

import random
import threading
import time

from line_profiler import LineProfiler


def do_other_stuff(numbers):
    time.sleep(1)
    s = sum(numbers)


def do_stuff(numbers):
    my_thread = threading.Thread(target=do_other_stuff, args=[numbers])
    my_thread.start()
    my_thread.join()


thread_to_lp = {}


def wrapper_in_thread(func):
    def inner(*args, **kwargs):
        lp = thread_to_lp.setdefault(threading.current_thread(), LineProfiler())
        lp.add_function(func)
        lp.enable_by_count()
        try:
            return func(*args, **kwargs)
        finally:
            lp.disable_by_count()

    return inner


do_other_stuff = wrapper_in_thread(do_other_stuff)
do_stuff = wrapper_in_thread(do_stuff)

numbers = [random.randint(1, 100) for i in range(1000)]
do_stuff(numbers)

for lp in thread_to_lp.values():
    lp.print_stats()