Python argparse appears in gc.garbage

2019-09-02 10:39发布

问题:

I'm trying to debug a memory leak in a python application and I can see a lot of non collected object belonging to the module argparse

Here a minimal script reproducing the error

import gc
gc.set_debug(gc.DEBUG_LEAK)

def get_cli_arguments():
    import argparse
    parser = argparse.ArgumentParser(description='Launch a RHM server')
    parser.add_argument(
        '--port',
        metavar='port',
        type=int,
        nargs='?',
        help='the server port',
        default=8003
    )
    return vars(parser.parse_args())


def main():
    args = get_cli_arguments()
    x = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"


main()
gc.collect()
print(gc.garbage)

and I got the following output

gc: collectable <dict 0x7f9a8b303c58>
gc: collectable <dict 0x7f9a8b303d70>
gc: collectable <list 0x7f9a8b350ea8>
gc: collectable <list 0x7f9a8b350b00>
gc: collectable <list 0x7f9a8b3dca70>
gc: collectable <_ArgumentGroup 0x7f9a8b3e6250>
gc: collectable <dict 0x7f9a8b301910>
gc: collectable <list 0x7f9a8b3535a8>
gc: collectable <list 0x7f9a8b3e1758>
gc: collectable <function 0x7f9a8b3d2d70>
gc: collectable <dict 0x7f9a8b301a28>
gc: collectable <list 0x7f9a8b3e17e8>
gc: collectable <_HelpAction 0x7f9a8b3e60d0>
gc: collectable <dict 0x7f9a8b3014b0>
gc: collectable <HelpFormatter 0x7f9a8b2f7f90>
gc: collectable <_Section 0x7f9a8b2f7fd0>
gc: collectable <dict 0x7f9a8b3016e0>
gc: collectable <list 0x7f9a8b350248>
gc: collectable <dict 0x7f9a8b3015c8>
gc: collectable <dict 0x7f9a8b303e88>
gc: collectable <list 0x7f9a8b3e1d40>
gc: collectable <_StoreAction 0x7f9a8b30a090>
gc: collectable <dict 0x7f9a8b2f9c58>
gc: collectable <HelpFormatter 0x7f9a8b30a0d0>
gc: collectable <_Section 0x7f9a8b30a110>
gc: collectable <dict 0x7f9a8b3017f8>
gc: collectable <list 0x7f9a8b36c200>
gc: collectable <dict 0x7f9a8b301b40>
[{'action': {'store_false': <class 'argparse._StoreFalseAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'help': <class 'argparse._HelpAction'>, None: <class 'argparse._StoreAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'count': <class 'argparse._CountAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'version': <class 'argparse._VersionAction'>, 'store': <class 'argparse._StoreAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'append': <class 'argparse._AppendAction'>}, 'type': {None: <function identity at 0x7f9a8b3d2d70>}}, {'store_false': <class 'argparse._StoreFalseAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'help': <class 'argparse._HelpAction'>, None: <class 'argparse._StoreAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'count': <class 'argparse._CountAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'version': <class 'argparse._VersionAction'>, 'store': <class 'argparse._StoreAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'append': <class 'argparse._AppendAction'>}, [_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port')], [], [], <argparse._ArgumentGroup object at 0x7f9a8b3e6250>, {'_mutually_exclusive_groups': [], '_negative_number_matcher': <_sre.SRE_Pattern object at 0x7f9a8b437290>, 'description': None, '_option_string_actions': {'--port': _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port'), '-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None)}, 'title': 'optional arguments', '_has_negative_number_optionals': [], '_defaults': {}, 'prefix_chars': '-', 'argument_default': None, '_registries': {'action': {'store_false': <class 'argparse._StoreFalseAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'help': <class 'argparse._HelpAction'>, None: <class 'argparse._StoreAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'count': <class 'argparse._CountAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'version': <class 'argparse._VersionAction'>, 'store': <class 'argparse._StoreAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'append': <class 'argparse._AppendAction'>}, 'type': {None: <function identity at 0x7f9a8b3d2d70>}}, '_group_actions': [_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port')], '_action_groups': [], 'conflict_handler': 'error', '_actions': [_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port')]}, [], [_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port')], <function identity at 0x7f9a8b3d2d70>, {None: <function identity at 0x7f9a8b3d2d70>}, ['-h', '--help'], _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), {'const': None, 'help': 'show this help message and exit', 'option_strings': ['-h', '--help'], 'dest': 'help', 'required': False, 'nargs': 0, 'choices': None, 'default': '==SUPPRESS==', 'container': <argparse._ArgumentGroup object at 0x7f9a8b3e6250>, 'type': None, 'metavar': None}, <argparse.HelpFormatter object at 0x7f9a8b2f7f90>, <argparse._Section object at 0x7f9a8b2f7fd0>, {'items': [], 'formatter': <argparse.HelpFormatter object at 0x7f9a8b2f7f90>, 'heading': None, 'parent': None}, [], {'_current_indent': 0, '_level': 0, '_indent_increment': 2, '_action_max_length': 0, '_max_help_position': 24, '_width': 78, '_root_section': <argparse._Section object at 0x7f9a8b2f7fd0>, '_long_break_matcher': <_sre.SRE_Pattern object at 0x7f9a8b33e258>, '_prog': 'prout.py', '_current_section': <argparse._Section object at 0x7f9a8b2f7fd0>, '_whitespace_matcher': <_sre.SRE_Pattern object at 0x7f9a8b347750>}, {'--port': _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port'), '-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, help='show this help message and exit', metavar=None)}, ['--port'], _StoreAction(option_strings=['--port'], dest='port', nargs='?', const=None, default=8003, type=<type 'int'>, choices=None, help='the server port', metavar='port'), {'const': None, 'help': 'the server port', 'option_strings': ['--port'], 'dest': 'port', 'required': False, 'nargs': '?', 'choices': None, 'default': 8003, 'container': <argparse._ArgumentGroup object at 0x7f9a8b3e6250>, 'type': <type 'int'>, 'metavar': 'port'}, <argparse.HelpFormatter object at 0x7f9a8b30a0d0>, <argparse._Section object at 0x7f9a8b30a110>, {'items': [], 'formatter': <argparse.HelpFormatter object at 0x7f9a8b30a0d0>, 'heading': None, 'parent': None}, [], {'_current_indent': 0, '_level': 0, '_indent_increment': 2, '_action_max_length': 0, '_max_help_position': 24, '_width': 78, '_root_section': <argparse._Section object at 0x7f9a8b30a110>, '_long_break_matcher': <_sre.SRE_Pattern object at 0x7f9a8b33e258>, '_prog': 'prout.py', '_current_section': <argparse._Section object at 0x7f9a8b30a110>, '_whitespace_matcher': <_sre.SRE_Pattern object at 0x7f9a8b347750>}]

I can't find any report of such a problem in argparse, am I doing something wrong?

回答1:

This line

gc.set_debug(gc.DEBUG_LEAK)

Is equivalent to saying

gc.set_debug(gc.DEBUG_COLLECTABLE | gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_SAVEALL)

And by setting DEBUG_SAVEALL,

When set, all unreachable objects found will be appended to garbage rather than being freed.

In fact, these would have been freed, had you not set DEBUG_LEAK. (Try your code it without setting DEBUG_LEAK)

The flag you likely want is gc.DEBUG_UNREACHABLE, which displays information about objects that are unreachable (and should probably be freed) but cannot be freed by the garbage collector.

You could also look at DEBUG_COLLECTABLE to help identify circular references that can be freed.