I'm trying to use multiprocessing.Pool
in my code but I got this exception:
PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup __builtin__.instancemethod failed
I found this and it's preferred solution recipe
my problem is that I don't know how to implement this solution in my code.
my code is something like that:
class G(class):
def submit(self,data):
cmd = self.createCommand(data)
subprocess.call(cmd, shell=True)
# call for a short command
def main(self):
self.pool = multiprocessing.Pool()
while(True):
data = self.GenerateData()
self.pool.apply_async(self.Submit, args=(data,))
some notes:
- the main
while
should work for a long time (few days)
- I'm using
pool
for performance purposes, if you have a better solution I will be glad to here it
update:
after using @unutbu solution I got the next exception:
PicklingError: Can't pickle <type 'thread.lock'>: attribute lookup thread.lock failed
now , all the solutions I found were talking about Queue.Queue
and mp.Pool.map
but I'm not using those attributes so I can't figure it out.
This is an application of Steven Bethard's solution to your situation:
import multiprocessing as mp
import time
import copy_reg
import types
def _pickle_method(method):
"""
Author: Steven Bethard
http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
"""
func_name = method.im_func.__name__
obj = method.im_self
cls = method.im_class
cls_name = ''
if func_name.startswith('__') and not func_name.endswith('__'):
cls_name = cls.__name__.lstrip('_')
if cls_name:
func_name = '_' + cls_name + func_name
return _unpickle_method, (func_name, obj, cls)
def _unpickle_method(func_name, obj, cls):
"""
Author: Steven Bethard
http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
"""
for cls in cls.mro():
try:
func = cls.__dict__[func_name]
except KeyError:
pass
else:
break
return func.__get__(obj, cls)
# This call to copy_reg.pickle allows you to pass methods as the first arg to
# mp.Pool methods. If you comment out this line, `pool.map(self.foo, ...)` results in
# PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup
# __builtin__.instancemethod failed
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
class G(object):
def submit(self, data):
print('processing {}'.format(data))
# cmd = self.createCommand(data)
# subprocess.call(cmd, shell=True)
# call for a short command
time.sleep(2)
def main(self):
pool = mp.Pool()
while True:
data = (1, 2, 3)
pool.apply_async(self.submit, args=(data,))
if __name__ == '__main__':
g = G()
g.main()