How to check if a multiprocessing.Lock() is locked?

Question

I'm trying to make a shared resource that can be accessed by multiple processes, but is limited in the number of requests it can handle at the same time. In the final product, this shared resource makes a data request based on an API.

As I'm going to make a lot of request I want to establish multiple separate connections to speed things up a little bit. In this case I want to establish three simultaneous connections and divide the requests over these connections.

The semaphore seems to work and limits the number of simultaneous function calls to three. However, I don't seem to be able to spread the load of the three calls over the different functions as the multiprocessing.Lock does not support the locked() method like its threading equivalent.

I was wondering if someone could help me to spread these three calls over the different initiated sessions or has a good suggestion on how to approach this problem in a different way?

Many, many thanks in advance!

import time

import multiprocessing
import concurrent.futures

from multiprocessing.managers import BaseManager

class Tester:
    def __init__(self, sessions=3):
        self.semaphore = multiprocessing.BoundedSemaphore(sessions)

        self.locka = multiprocessing.Lock()
        self.lockb = multiprocessing.Lock()
        self.lockc = multiprocessing.Lock()

    def call(self, name):
        with self.semaphore:

            while True:
                if not self.locka.locked():
                    with self.locka:
                        time.sleep(1)
                        return self.session_a(name)

                if not self.lockb.locked():    
                    with self.lockb:
                        time.sleep(1)
                        return self.session_b(name)

                if not self.lockc.locked():
                    with self.lockc:
                        time.sleep(1)
                        return self.session_c(name) 

    def session_a(self, name):
        print(f'session_a:  {name}')

    def session_b(self, name):
        print(f'session_b:  {name}')

    def session_c(self):
        print(f'session_c:  {name}')


def call_object(obj, name):
    obj.call(name)


def main():
    BaseManager.register('Tester', Tester)
    manager = BaseManager()

    manager.start()
    inst = manager.Tester()

    with concurrent.futures.ProcessPoolExecutor() as executor:
        names = ['Alex', 'Brain', 'Carl', 'Derek', 'Eric', 'Frank', 'George', 'Harold']
        futures = [executor.submit(call_object, inst, name) for name in names]


if __name__ == "__main__":
    main()

Answer 1

Use locks only and:

• cycle through locks/sessions till a lock is acquired and
• a result is obtained
• return the result

When trying to acquire a lock don't block so you can try the next one - it will return True if acquired, False if not acquired.

---

import time
import multiprocessing
import concurrent.futures

from multiprocessing.managers import BaseManager

class Tester:
    def __init__(self, sessions=3):
        self.locks = [multiprocessing.Lock(),
                      multiprocessing.Lock(),
                      multiprocessing.Lock()]

    def call(self, name):
        # cycle through locks/sessions till a lock is acquired and
        # a result is obtained
        # return the result
        done = False
        while not done:
            for lock,session in zip(self.locks,
                                    [self.session_a,
                                     self.session_b,
                                     self.session_c]):
                acq = lock.acquire(block=False)
                if acq:
                    #print(f'lock acquired for {session.__name__}:{name}',file=sys.stdout)
                    try:
                        time.sleep(1)
                        result = session(name)
                    finally:
                        lock.release()
                        #print(f'lock for {session.__name__} released')
                        done = True
                    #print(result)
                    break
        return result + '!!'


    def session_a(self, name):
        #print(f'in method session_a:  {name}')
        return f'session_a:  {name}'

    def session_b(self, name):
        #print(f'in method session_b:  {name}')
        return f'session_b:  {name}'

    def session_c(self,name):
        #print(f'in method session_c:  {name}')
        return f'session_c:  {name}'

def call_object(obj, name):
    return obj.call(name)

def main():
    BaseManager.register('Tester', Tester)
    manager = BaseManager()

    manager.start()
    inst = manager.Tester()

    with concurrent.futures.ProcessPoolExecutor() as executor:
        names = ['Alex', 'Brain', 'Carl', 'Derek', 'Eric', 'Frank', 'George', 'Harold']
        futures = [executor.submit(call_object, inst, name) for name in names]
        for fut in concurrent.futures.as_completed(futures):
            print(f'future result {fut.result()}')