Multiprocessing pool apply_async function when applied to an Object throws Cannot pickle <type 'thread.lock'> objects

Question

My __init__ function tries to call a function of the Scheduler class called verify_func() .

def__init__:
def _pickle_method(method):
    func_name = method.im_func.__name__
    obj = method.im_self
    cls = method.im_class
    return _unpickle_method, (func_name, obj, cls)

def _unpickle_method(func_name, obj, cls):
    for cls in cls.mro():
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
    return func.__get__(obj, cls)


if __name__=='__main__':
  while True:
    #Create a scheduler object
    scheduler_obj=Scheduler()
    try:
      #Connect to the database get all the new requests to be verified
      db = Database(scheduler_obj.username_testschema, scheduler_obj.password_testschema, scheduler_obj.mother_host_testschema,
                    scheduler_obj.mother_port_testschema, scheduler_obj.mother_sid_testschema, 0)
      result = db.query("SELECT JOB_ID FROM %s.table_1 WHERE JOB_STATUS='%s'" % (scheduler_obj.username_testschema, 'initiated'))
      copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
      pool = mp.Pool(len(result))
      for row in result:
        verify_id = row[0]
        print mp.current_process()
        try:
          pool.apply_async(scheduler_obj.verify_func, (verify_id))
        except Exception as e:
                    scheduler_obj.logger.exception("MP exception : %s", e)

      pool.close()
      pool.join()

    except Exception as e:
      scheduler_obj.logger.exception(e)
    finally:
      time.sleep(10)

This verify_func tries to run some queries on the database using cx_oracle.

def verify_func(self,verify_id):
    print "Verifying with Current Id : ", verify_id
    try:
      db = Database(self.username_testschema, self.password_testschema, self.mother_host_testschema,
                    self.mother_port_testschema, self.mother_sid_testschema, 0)
      result = db.query("select * from %s.table_2 where job_id=%d" % (self.username_testschema,verify_id))
      column_name = [d[0] for d in db.cursor.description]
      final_dictionary = [dict(zip(column_name, row)) for row in result]

      #On getting the new job_ids whose status is .. try to run checks for each one.
      for row in final_dictionary:
        print row

So is it connecting and running a query on the database, which throws this error as it doesn't say where exactly it is breaking.

class Database(object):
 def __init__:
    self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode) 

Let me know if more explanation is needed.

Answer 1

Ya i did a scheduler_object. dict and there i saw i have a logger instance variable, which is not pickleable. Thanks @Dano.

So the best way to debug is find out what can be the class variable/ method coz if which you are getting this error. Also, you can override logger class` getstate , setstate methods to make it pickleable as shown here. Can't pickle loggers?