using multiprocessing.Pool on bound methods

Question

I'm trying to use multiprocessing.Pool in my code but I got this exception:

PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup __builtin__.instancemethod failed

I found this and it's preferred solution recipe

my problem is that I don't know how to implement this solution in my code.

my code is something like that:

class G(class):
    def submit(self,data):
        cmd = self.createCommand(data)
        subprocess.call(cmd, shell=True)
        # call for a short command

    def main(self):
        self.pool = multiprocessing.Pool()
        while(True):
            data = self.GenerateData()
            self.pool.apply_async(self.Submit, args=(data,))

some notes:

• the main while should work for a long time (few days)
• I'm using pool for performance purposes, if you have a better solution I will be glad to here it

update:

after using @unutbu solution I got the next exception: PicklingError: Can't pickle <type 'thread.lock'>: attribute lookup thread.lock failed

now , all the solutions I found were talking about Queue.Queue and mp.Pool.map but I'm not using those attributes so I can't figure it out.

Answer 1

This is an application of Steven Bethard's solution to your situation:

import multiprocessing as mp
import time
import copy_reg
import types

def _pickle_method(method):
    """
    Author: Steven Bethard 
    http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
    """
    func_name = method.im_func.__name__
    obj = method.im_self
    cls = method.im_class
    cls_name = ''
    if func_name.startswith('__') and not func_name.endswith('__'):
        cls_name = cls.__name__.lstrip('_')
    if cls_name:
        func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)


def _unpickle_method(func_name, obj, cls):
    """
    Author: Steven Bethard
    http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
    """
    for cls in cls.mro():
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls)

# This call to copy_reg.pickle allows you to pass methods as the first arg to
# mp.Pool methods. If you comment out this line, `pool.map(self.foo, ...)` results in
# PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup
# __builtin__.instancemethod failed

copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)

class G(object):
    def submit(self, data):
        print('processing {}'.format(data))
        # cmd = self.createCommand(data)
        # subprocess.call(cmd, shell=True)
        # call for a short command
        time.sleep(2)

    def main(self):
        pool = mp.Pool()
        while True:
            data = (1, 2, 3)
            pool.apply_async(self.submit, args=(data,))

if __name__ == '__main__':
    g = G()
    g.main()