How to send a function to a remote Pyro object

Question

I am trying to set up some code using Pyro to process python code functions on a remote host and get results back. After starting the name server, i would execute this code on the remote host (actually still on localhost):

import Pyro4

class Server(object):
    def evaluate(self, func, args):
        return func(*args)

def main():
    server = Server()
    Pyro4.Daemon.serveSimple(
            {
                server: "server"
            },
            ns=True)

if __name__ == '__main__':
    main()

On the client side i have this code, which is an example of the behaviour i am trying to set up.

import Pyro4

remoteServer = Pyro4.Proxy('PYRONAME:server')

def square(x): 
    return x**2

print remoteServer.evaluate(square, 4)

However, this code results in the following exception:

/usr/lib/python2.7/site-packages/Pyro4/core.py:155: UserWarning: HMAC_KEY not set,
protocol data may not be secure
warnings.warn("HMAC_KEY not set, protocol data may not be secure")
Traceback (most recent call last):
  File "/home/davide/Projects/rempy/example-api-pyro.py", line 7, in <module>
    print remoteServer.evaluate(square, 4)
  File "/usr/lib/python2.7/site-packages/Pyro4/core.py", line 149, in __call__
    return self.__send(self.__name, args, kwargs)
  File "/usr/lib/python2.7/site-packages/Pyro4/core.py", line 289, in _pyroInvoke
    raise data
AttributeError: 'module' object has no attribute 'square'

It seems to me that the function object is pickled correctly and is sent to the Server instance on the remote host, but there is some problem in the namespace.

How can i solve this problem?

Thanks

Answer 1

I think i know your problem:

the module the function is defiined in is called

'__main__'

it exists in all running versions of python.

pickle does not transfer the source code but a reference

__main__.square

so you have two possibilities:

source square out and make the main module as short as possible such as:

# main.py

def square(x): 
   return x**2

import Pyro4
def main():
    remoteServer = Pyro4.Proxy('PYRONAME:server')


    print remoteServer.evaluate(square, 4)

and:

# __main__.py
import main
main.main()

Then the server can import exactly the same module from the file.

or create a module with my code:

class ThisShallNeverBeCalledError(Exception):
    pass

class _R(object):
    def __init__(self, f, *args):
        self.ret = (f, args)
    def __reduce__(self):
        return self.ret
    def __call__(self, *args):
        raise ThisShallNeverBeCalledError()

    @classmethod
    def fromReduce(cls, value):
        ret = cls(None)
        ret.ret = value
        return ret


def dump_and_load(obj):
    '''pickle and unpickle the object once'''
    s = pickle.dumps(obj)
    return pickle.loads(s)

# this string creates an object of an anonymous type that can
# be called to create an R object or that can be reduced by pickle
# and creates another anonymous type when unpickled
# you may not inherit from this MetaR object because it is not a class
PICKLABLE_R_STRING= "type('MetaR', (object,), " \
                    "       {'__call__' : lambda self, f, *args: "\
                    "          type('PICKLABLE_R', "\
                    "               (object,), "\
                    "               {'__reduce__' : lambda self: (f, args), "\
                    "                '__module__' : 'pickleHelp_', "\
                    "                '__name__'   : 'PICKLABLE_R', "\
                    "                '__call__'   : lambda self: None})(), "\
                    "        '__reduce__' : lambda self: "\
                    "           self(eval, meta_string, "\
                    "                {'meta_string' : meta_string}).__reduce__(), "\
                    "        '__module__' : 'pickleHelp_', "\
                    "        '__name__' : 'R'})()".replace('  ', '')
PICKLABLE_R = _R(eval, PICKLABLE_R_STRING, \
                {'meta_string' : PICKLABLE_R_STRING})
R = dump_and_load(PICKLABLE_R)
del PICKLABLE_R, PICKLABLE_R_STRING

PICKLABLE___builtins__ = R(vars, R(__import__, '__builtin__'))
PICKLABLE_FunctionType = R(type, R(eval, 'lambda:None'))

##R.__module__ = __name__
##R.__name__ = 'PICKLABLE_R'


def packCode(code, globals = {}, add_builtins = True, use_same_globals = False, \
             check_syntax = True, return_value_variable_name = 'obj',
             __name__ = __name__ + '.packCode()'):
    '''return an object that executes code in globals when unpickled
use_same_globals
    if use_same_globals is True all codes sent through
    one pickle connection share the same globals
    by default the dont
return_value_variable_name
    if a variable with the name in return_value_variable_name exists
    in globals after the code execution
    it is returned as result of the pickling operation
    if not None is returned
__name__

'''
    if check_syntax:
        compile(code, '', 'exec')
    # copying locals is important
    # locals is transferred through pickle for all code identical
    # copying it prevents different code from beeing executed in same globals
    if not use_same_globals:
        globals = globals.copy()
    if add_builtins:
        globals['__builtins__'] = PICKLABLE___builtins__
    globals.setdefault('obj', None)
    # get the compilation code
    # do not marshal or unmarshal code objects because the platforms may vary
    code = R(compile, code, __name__, 'exec')
    # the final object that can reduce, dump and load itself
    obj = R(R(getattr, tuple, '__getitem__'), (
            R(R(PICKLABLE_FunctionType, code, globals)),
            R(R(getattr, type(globals), 'get'), globals, \
              returnValueVariableName, None)
            ), -1)
    return obj

and then send this to the other side:

packCode('''
def square(...):
    ...
''', return_value_variable_name = 'square')

and the function will come out on the other side, no module code is needed to transefer this python function to the other server side.

If something does not work out please tell me.