Based on the example multiprocess using pathos and dill I hit the wall of cPickle errors. I did as www3cam did; with a little addition.. by removing pathos, dill, multiprocess and pyreadline. Then hit pip install pathos --no-cache-dir after carefully removed those side-packages mentioned above. Fresh install, cold-reboot of PC and an Anacoda2 update later... the jar with cPickles... is still there and Frank Zappa keeps singing his "Titties and Beer" song.
The modified code from 1:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os, sys
import datetime, time
import multiprocessing
import pathos.pools as pp
import dill
class Multiprocess(object):
def __init__(self):
pass
def qmp_worker(self,(inputs, the_time)):
print " Processs %s\tWaiting %s seconds" % (inputs, the_time)
time.sleep(int(the_time))
print " Process %s\tDONE" % inputs
def qmp_handler(self): # Non tandem pair processing
pool = pp.ProcessPool(2)
pool.map(self.qmp_worker, data)
def mp_worker((inputs, the_time)):
print " Process %s\tWaiting %s seconds" % (inputs, the_time)
time.sleep(int(the_time))
print " Process %s\tDONE" % inputs
sys.stdout.flush()
def mp_handler(): # Non tandem pair processing
p = multiprocessing.Pool(2)
p.map(mp_worker, data)
def mp_handler_tandem():
subdata = zip(data[0::2], data[1::2])
# print subdata
for task1, task2 in subdata:
p = multiprocessing.Pool(2)
p.map(mp_worker, (task1, task2))
#data = (['a', '1'], ['b', '2'], ['c', '3'], ['d', '4'])
data = (['a', '2'], ['b', '3'], ['c', '1'], ['d', '4'],
['e', '1'], ['f', '2'], ['g', '3'], ['h', '4'])
if __name__ == '__main__':
sys.stdout.flush()
# print 'mp_handler():'
# mp_handler()
# print '---'
# time.sleep(2)
# print '\nmp_handler_tandem():'
# mp_handler_tandem()
print '---'
# time.sleep(2)
Multiprocess().qmp_handler()
Perhaps something should be set like with the example from here "catching and printing PyQt5 tracebacks?
Snippet code below that made me thinking and curious if something like that has to be done done for dill?
# Back up the reference to the exceptionhook
sys._excepthook = sys.excepthook
# Set the exception hook to our wrapping function
sys.excepthook = my_exception_hook
Latest error:
File "c:\python\anac2\lib\site-packages\pathos\multiprocessing.py", line 137, in map return _pool.map(star(f), zip(*args)) # chunksize File "c:\python\anac2\lib\site-packages\multiprocess\pool.py", line 251, in map return self.map_async(func, iterable, chunksize).get() File "c:\python\anac2\lib\site-packages\multiprocess\pool.py", line 567, in get raise self._value cPickle.PicklingError: Can't pickle : attribute lookup builtin.function failed
More info about this error is at: my cPickle dump.
