|
|
05485cd |
From 59683c433fd4c19f4921e4b44b8a70174e82fc4b Mon Sep 17 00:00:00 2001
|
|
|
05485cd |
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
|
|
05485cd |
Date: Tue, 10 Sep 2019 08:32:24 +0200
|
|
|
05485cd |
Subject: [PATCH 18/18] Unbundle cloudpickle
|
|
|
05485cd |
|
|
|
05485cd |
---
|
|
|
05485cd |
joblib/externals/cloudpickle/__init__.py | 11 -
|
|
|
05485cd |
joblib/externals/cloudpickle/cloudpickle.py | 1380 ------------------
|
|
|
05485cd |
joblib/externals/loky/backend/reduction.py | 4 +-
|
|
|
05485cd |
joblib/externals/loky/cloudpickle_wrapper.py | 2 +-
|
|
|
05485cd |
joblib/parallel.py | 2 +-
|
|
|
05485cd |
setup.py | 2 +-
|
|
|
05485cd |
6 files changed, 5 insertions(+), 1396 deletions(-)
|
|
|
05485cd |
delete mode 100644 joblib/externals/cloudpickle/__init__.py
|
|
|
05485cd |
delete mode 100644 joblib/externals/cloudpickle/cloudpickle.py
|
|
|
05485cd |
|
|
|
05485cd |
diff --git a/joblib/externals/cloudpickle/__init__.py b/joblib/externals/cloudpickle/__init__.py
|
|
|
05485cd |
deleted file mode 100644
|
|
|
05485cd |
index 9806b5c7b2..0000000000
|
|
|
05485cd |
--- a/joblib/externals/cloudpickle/__init__.py
|
|
|
05485cd |
+++ /dev/null
|
|
|
05485cd |
@@ -1,11 +0,0 @@
|
|
|
05485cd |
-from __future__ import absolute_import
|
|
|
05485cd |
-
|
|
|
05485cd |
-import sys
|
|
|
05485cd |
-import pickle
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-from .cloudpickle import *
|
|
|
05485cd |
-if sys.version_info[:2] >= (3, 8):
|
|
|
05485cd |
- from .cloudpickle_fast import CloudPickler, dumps, dump
|
|
|
05485cd |
-
|
|
|
05485cd |
-__version__ = '1.2.1'
|
|
|
05485cd |
diff --git a/joblib/externals/cloudpickle/cloudpickle.py b/joblib/externals/cloudpickle/cloudpickle.py
|
|
|
05485cd |
deleted file mode 100644
|
|
|
05485cd |
index cc74c642ec..0000000000
|
|
|
05485cd |
--- a/joblib/externals/cloudpickle/cloudpickle.py
|
|
|
05485cd |
+++ /dev/null
|
|
|
05485cd |
@@ -1,1380 +0,0 @@
|
|
|
775c617 |
-"""
|
|
|
775c617 |
-This class is defined to override standard pickle functionality
|
|
|
775c617 |
-
|
|
|
775c617 |
-The goals of it follow:
|
|
|
775c617 |
--Serialize lambdas and nested functions to compiled byte code
|
|
|
775c617 |
--Deal with main module correctly
|
|
|
775c617 |
--Deal with other non-serializable objects
|
|
|
775c617 |
-
|
|
|
775c617 |
-It does not include an unpickler, as standard python unpickling suffices.
|
|
|
775c617 |
-
|
|
|
775c617 |
-This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
|
|
|
775c617 |
-<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
|
|
|
775c617 |
-
|
|
|
775c617 |
-Copyright (c) 2012, Regents of the University of California.
|
|
|
775c617 |
-Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
|
|
|
775c617 |
-All rights reserved.
|
|
|
775c617 |
-
|
|
|
775c617 |
-Redistribution and use in source and binary forms, with or without
|
|
|
775c617 |
-modification, are permitted provided that the following conditions
|
|
|
775c617 |
-are met:
|
|
|
775c617 |
- * Redistributions of source code must retain the above copyright
|
|
|
775c617 |
- notice, this list of conditions and the following disclaimer.
|
|
|
775c617 |
- * Redistributions in binary form must reproduce the above copyright
|
|
|
775c617 |
- notice, this list of conditions and the following disclaimer in the
|
|
|
775c617 |
- documentation and/or other materials provided with the distribution.
|
|
|
775c617 |
- * Neither the name of the University of California, Berkeley nor the
|
|
|
775c617 |
- names of its contributors may be used to endorse or promote
|
|
|
775c617 |
- products derived from this software without specific prior written
|
|
|
775c617 |
- permission.
|
|
|
775c617 |
-
|
|
|
775c617 |
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
775c617 |
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
775c617 |
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
775c617 |
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
775c617 |
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
775c617 |
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
|
|
775c617 |
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
|
775c617 |
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
|
775c617 |
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
|
775c617 |
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
|
775c617 |
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
775c617 |
-"""
|
|
|
775c617 |
-from __future__ import print_function
|
|
|
775c617 |
-
|
|
|
775c617 |
-import dis
|
|
|
775c617 |
-from functools import partial
|
|
|
775c617 |
-import io
|
|
|
775c617 |
-import itertools
|
|
|
775c617 |
-import logging
|
|
|
775c617 |
-import opcode
|
|
|
775c617 |
-import operator
|
|
|
775c617 |
-import pickle
|
|
|
05485cd |
-import platform
|
|
|
775c617 |
-import struct
|
|
|
775c617 |
-import sys
|
|
|
775c617 |
-import traceback
|
|
|
775c617 |
-import types
|
|
|
775c617 |
-import weakref
|
|
|
05485cd |
-import uuid
|
|
|
05485cd |
-import threading
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-try:
|
|
|
05485cd |
- from enum import Enum
|
|
|
05485cd |
-except ImportError:
|
|
|
05485cd |
- Enum = None
|
|
|
775c617 |
-
|
|
|
775c617 |
-# cloudpickle is meant for inter process communication: we expect all
|
|
|
775c617 |
-# communicating processes to run the same Python version hence we favor
|
|
|
775c617 |
-# communication speed over compatibility:
|
|
|
775c617 |
-DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
|
|
|
775c617 |
-
|
|
|
05485cd |
-# Track the provenance of reconstructed dynamic classes to make it possible to
|
|
|
05485cd |
-# recontruct instances from the matching singleton class definition when
|
|
|
05485cd |
-# appropriate and preserve the usual "isinstance" semantics of Python objects.
|
|
|
05485cd |
-_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
|
|
|
05485cd |
-_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
|
|
|
05485cd |
-_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
|
|
|
05485cd |
-
|
|
|
05485cd |
-PYPY = platform.python_implementation() == "PyPy"
|
|
|
05485cd |
-
|
|
|
05485cd |
-builtin_code_type = None
|
|
|
05485cd |
-if PYPY:
|
|
|
05485cd |
- # builtin-code objects only exist in pypy
|
|
|
05485cd |
- builtin_code_type = type(float.__new__.__code__)
|
|
|
775c617 |
-
|
|
|
775c617 |
-if sys.version_info[0] < 3: # pragma: no branch
|
|
|
775c617 |
- from pickle import Pickler
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- from cStringIO import StringIO
|
|
|
775c617 |
- except ImportError:
|
|
|
775c617 |
- from StringIO import StringIO
|
|
|
775c617 |
- string_types = (basestring,) # noqa
|
|
|
775c617 |
- PY3 = False
|
|
|
05485cd |
- PY2 = True
|
|
|
775c617 |
-else:
|
|
|
775c617 |
- types.ClassType = type
|
|
|
775c617 |
- from pickle import _Pickler as Pickler
|
|
|
775c617 |
- from io import BytesIO as StringIO
|
|
|
775c617 |
- string_types = (str,)
|
|
|
775c617 |
- PY3 = True
|
|
|
05485cd |
- PY2 = False
|
|
|
05485cd |
- from importlib._bootstrap import _find_spec
|
|
|
05485cd |
-
|
|
|
05485cd |
-_extract_code_globals_cache = weakref.WeakKeyDictionary()
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _ensure_tracking(class_def):
|
|
|
05485cd |
- with _DYNAMIC_CLASS_TRACKER_LOCK:
|
|
|
05485cd |
- class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
|
|
|
05485cd |
- if class_tracker_id is None:
|
|
|
05485cd |
- class_tracker_id = uuid.uuid4().hex
|
|
|
05485cd |
- _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
|
|
|
05485cd |
- _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
|
|
|
05485cd |
- return class_tracker_id
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _lookup_class_or_track(class_tracker_id, class_def):
|
|
|
05485cd |
- if class_tracker_id is not None:
|
|
|
05485cd |
- with _DYNAMIC_CLASS_TRACKER_LOCK:
|
|
|
05485cd |
- class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
|
|
|
05485cd |
- class_tracker_id, class_def)
|
|
|
05485cd |
- _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
|
|
|
05485cd |
- return class_def
|
|
|
05485cd |
-
|
|
|
05485cd |
-if sys.version_info[:2] >= (3, 5):
|
|
|
05485cd |
- from pickle import _getattribute
|
|
|
05485cd |
-elif sys.version_info[:2] >= (3, 4):
|
|
|
05485cd |
- from pickle import _getattribute as _py34_getattribute
|
|
|
05485cd |
- # pickle._getattribute does not return the parent under Python 3.4
|
|
|
05485cd |
- def _getattribute(obj, name):
|
|
|
05485cd |
- return _py34_getattribute(obj, name), None
|
|
|
05485cd |
-else:
|
|
|
05485cd |
- # pickle._getattribute is a python3 addition and enchancement of getattr,
|
|
|
05485cd |
- # that can handle dotted attribute names. In cloudpickle for python2,
|
|
|
05485cd |
- # handling dotted names is not needed, so we simply define _getattribute as
|
|
|
05485cd |
- # a wrapper around getattr.
|
|
|
05485cd |
- def _getattribute(obj, name):
|
|
|
05485cd |
- return getattr(obj, name, None), None
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _whichmodule(obj, name):
|
|
|
05485cd |
- """Find the module an object belongs to.
|
|
|
05485cd |
-
|
|
|
05485cd |
- This function differs from ``pickle.whichmodule`` in two ways:
|
|
|
05485cd |
- - it does not mangle the cases where obj's module is __main__ and obj was
|
|
|
05485cd |
- not found in any module.
|
|
|
05485cd |
- - Errors arising during module introspection are ignored, as those errors
|
|
|
05485cd |
- are considered unwanted side effects.
|
|
|
05485cd |
- """
|
|
|
05485cd |
- module_name = getattr(obj, '__module__', None)
|
|
|
05485cd |
- if module_name is not None:
|
|
|
05485cd |
- return module_name
|
|
|
05485cd |
- # Protect the iteration by using a list copy of sys.modules against dynamic
|
|
|
05485cd |
- # modules that trigger imports of other modules upon calls to getattr.
|
|
|
05485cd |
- for module_name, module in list(sys.modules.items()):
|
|
|
05485cd |
- if module_name == '__main__' or module is None:
|
|
|
05485cd |
- continue
|
|
|
05485cd |
- try:
|
|
|
05485cd |
- if _getattribute(module, name)[0] is obj:
|
|
|
05485cd |
- return module_name
|
|
|
05485cd |
- except Exception:
|
|
|
05485cd |
- pass
|
|
|
05485cd |
- return None
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _is_global(obj, name=None):
|
|
|
05485cd |
- """Determine if obj can be pickled as attribute of a file-backed module"""
|
|
|
05485cd |
- if name is None:
|
|
|
05485cd |
- name = getattr(obj, '__qualname__', None)
|
|
|
05485cd |
- if name is None:
|
|
|
05485cd |
- name = getattr(obj, '__name__', None)
|
|
|
05485cd |
-
|
|
|
05485cd |
- module_name = _whichmodule(obj, name)
|
|
|
05485cd |
-
|
|
|
05485cd |
- if module_name is None:
|
|
|
05485cd |
- # In this case, obj.__module__ is None AND obj was not found in any
|
|
|
05485cd |
- # imported module. obj is thus treated as dynamic.
|
|
|
05485cd |
- return False
|
|
|
05485cd |
-
|
|
|
05485cd |
- if module_name == "__main__":
|
|
|
05485cd |
- return False
|
|
|
05485cd |
-
|
|
|
05485cd |
- module = sys.modules.get(module_name, None)
|
|
|
05485cd |
- if module is None:
|
|
|
05485cd |
- # The main reason why obj's module would not be imported is that this
|
|
|
05485cd |
- # module has been dynamically created, using for example
|
|
|
05485cd |
- # types.ModuleType. The other possibility is that module was removed
|
|
|
05485cd |
- # from sys.modules after obj was created/imported. But this case is not
|
|
|
05485cd |
- # supported, as the standard pickle does not support it either.
|
|
|
05485cd |
- return False
|
|
|
05485cd |
-
|
|
|
05485cd |
- # module has been added to sys.modules, but it can still be dynamic.
|
|
|
05485cd |
- if _is_dynamic(module):
|
|
|
05485cd |
- return False
|
|
|
05485cd |
-
|
|
|
05485cd |
- try:
|
|
|
05485cd |
- obj2, parent = _getattribute(module, name)
|
|
|
05485cd |
- except AttributeError:
|
|
|
05485cd |
- # obj was not found inside the module it points to
|
|
|
05485cd |
- return False
|
|
|
05485cd |
- return obj2 is obj
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _extract_code_globals(co):
|
|
|
05485cd |
- """
|
|
|
05485cd |
- Find all globals names read or written to by codeblock co
|
|
|
05485cd |
- """
|
|
|
05485cd |
- out_names = _extract_code_globals_cache.get(co)
|
|
|
05485cd |
- if out_names is None:
|
|
|
05485cd |
- names = co.co_names
|
|
|
05485cd |
- out_names = {names[oparg] for _, oparg in _walk_global_ops(co)}
|
|
|
05485cd |
-
|
|
|
05485cd |
- # Declaring a function inside another one using the "def ..."
|
|
|
05485cd |
- # syntax generates a constant code object corresonding to the one
|
|
|
05485cd |
- # of the nested function's As the nested function may itself need
|
|
|
05485cd |
- # global variables, we need to introspect its code, extract its
|
|
|
05485cd |
- # globals, (look for code object in it's co_consts attribute..) and
|
|
|
05485cd |
- # add the result to code_globals
|
|
|
05485cd |
- if co.co_consts:
|
|
|
05485cd |
- for const in co.co_consts:
|
|
|
05485cd |
- if isinstance(const, types.CodeType):
|
|
|
05485cd |
- out_names |= _extract_code_globals(const)
|
|
|
05485cd |
-
|
|
|
05485cd |
- _extract_code_globals_cache[co] = out_names
|
|
|
05485cd |
-
|
|
|
05485cd |
- return out_names
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
05485cd |
-def _find_imported_submodules(code, top_level_dependencies):
|
|
|
05485cd |
- """
|
|
|
05485cd |
- Find currently imported submodules used by a function.
|
|
|
05485cd |
-
|
|
|
05485cd |
- Submodules used by a function need to be detected and referenced for the
|
|
|
05485cd |
- function to work correctly at depickling time. Because submodules can be
|
|
|
05485cd |
- referenced as attribute of their parent package (``package.submodule``), we
|
|
|
05485cd |
- need a special introspection technique that does not rely on GLOBAL-related
|
|
|
05485cd |
- opcodes to find references of them in a code object.
|
|
|
05485cd |
-
|
|
|
05485cd |
- Example:
|
|
|
05485cd |
- ```
|
|
|
05485cd |
- import concurrent.futures
|
|
|
05485cd |
- import cloudpickle
|
|
|
05485cd |
- def func():
|
|
|
05485cd |
- x = concurrent.futures.ThreadPoolExecutor
|
|
|
05485cd |
- if __name__ == '__main__':
|
|
|
05485cd |
- cloudpickle.dumps(func)
|
|
|
05485cd |
- ```
|
|
|
05485cd |
- The globals extracted by cloudpickle in the function's state include the
|
|
|
05485cd |
- concurrent package, but not its submodule (here, concurrent.futures), which
|
|
|
05485cd |
- is the module used by func. Find_imported_submodules will detect the usage
|
|
|
05485cd |
- of concurrent.futures. Saving this module alongside with func will ensure
|
|
|
05485cd |
- that calling func once depickled does not fail due to concurrent.futures
|
|
|
05485cd |
- not being imported
|
|
|
05485cd |
- """
|
|
|
05485cd |
-
|
|
|
05485cd |
- subimports = []
|
|
|
05485cd |
- # check if any known dependency is an imported package
|
|
|
05485cd |
- for x in top_level_dependencies:
|
|
|
05485cd |
- if (isinstance(x, types.ModuleType) and
|
|
|
05485cd |
- hasattr(x, '__package__') and x.__package__):
|
|
|
05485cd |
- # check if the package has any currently loaded sub-imports
|
|
|
05485cd |
- prefix = x.__name__ + '.'
|
|
|
05485cd |
- # A concurrent thread could mutate sys.modules,
|
|
|
05485cd |
- # make sure we iterate over a copy to avoid exceptions
|
|
|
05485cd |
- for name in list(sys.modules):
|
|
|
05485cd |
- # Older versions of pytest will add a "None" module to
|
|
|
05485cd |
- # sys.modules.
|
|
|
05485cd |
- if name is not None and name.startswith(prefix):
|
|
|
05485cd |
- # check whether the function can address the sub-module
|
|
|
05485cd |
- tokens = set(name[len(prefix):].split('.'))
|
|
|
05485cd |
- if not tokens - set(code.co_names):
|
|
|
05485cd |
- subimports.append(sys.modules[name])
|
|
|
05485cd |
- return subimports
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _make_cell_set_template_code():
|
|
|
775c617 |
- """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
|
|
|
775c617 |
-
|
|
|
775c617 |
- Notes
|
|
|
775c617 |
- -----
|
|
|
775c617 |
- In Python 3, we could use an easier function:
|
|
|
775c617 |
-
|
|
|
775c617 |
- .. code-block:: python
|
|
|
775c617 |
-
|
|
|
775c617 |
- def f():
|
|
|
775c617 |
- cell = None
|
|
|
775c617 |
-
|
|
|
775c617 |
- def _stub(value):
|
|
|
775c617 |
- nonlocal cell
|
|
|
775c617 |
- cell = value
|
|
|
775c617 |
-
|
|
|
775c617 |
- return _stub
|
|
|
775c617 |
-
|
|
|
775c617 |
- _cell_set_template_code = f().__code__
|
|
|
775c617 |
-
|
|
|
775c617 |
- This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is
|
|
|
775c617 |
- invalid syntax on Python 2. If we use this function we also don't need
|
|
|
775c617 |
- to do the weird freevars/cellvars swap below
|
|
|
775c617 |
- """
|
|
|
775c617 |
- def inner(value):
|
|
|
775c617 |
- lambda: cell # make ``cell`` a closure so that we get a STORE_DEREF
|
|
|
775c617 |
- cell = value
|
|
|
775c617 |
-
|
|
|
775c617 |
- co = inner.__code__
|
|
|
775c617 |
-
|
|
|
775c617 |
- # NOTE: we are marking the cell variable as a free variable intentionally
|
|
|
775c617 |
- # so that we simulate an inner function instead of the outer function. This
|
|
|
775c617 |
- # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
|
|
|
05485cd |
- if PY2: # pragma: no branch
|
|
|
775c617 |
- return types.CodeType(
|
|
|
775c617 |
- co.co_argcount,
|
|
|
775c617 |
- co.co_nlocals,
|
|
|
775c617 |
- co.co_stacksize,
|
|
|
775c617 |
- co.co_flags,
|
|
|
775c617 |
- co.co_code,
|
|
|
775c617 |
- co.co_consts,
|
|
|
775c617 |
- co.co_names,
|
|
|
775c617 |
- co.co_varnames,
|
|
|
775c617 |
- co.co_filename,
|
|
|
775c617 |
- co.co_name,
|
|
|
775c617 |
- co.co_firstlineno,
|
|
|
775c617 |
- co.co_lnotab,
|
|
|
775c617 |
- co.co_cellvars, # this is the trickery
|
|
|
775c617 |
- (),
|
|
|
775c617 |
- )
|
|
|
775c617 |
- else:
|
|
|
05485cd |
- if hasattr(types.CodeType, "co_posonlyargcount"): # pragma: no branch
|
|
|
05485cd |
- return types.CodeType(
|
|
|
05485cd |
- co.co_argcount,
|
|
|
05485cd |
- co.co_posonlyargcount, # Python3.8 with PEP570
|
|
|
05485cd |
- co.co_kwonlyargcount,
|
|
|
05485cd |
- co.co_nlocals,
|
|
|
05485cd |
- co.co_stacksize,
|
|
|
05485cd |
- co.co_flags,
|
|
|
05485cd |
- co.co_code,
|
|
|
05485cd |
- co.co_consts,
|
|
|
05485cd |
- co.co_names,
|
|
|
05485cd |
- co.co_varnames,
|
|
|
05485cd |
- co.co_filename,
|
|
|
05485cd |
- co.co_name,
|
|
|
05485cd |
- co.co_firstlineno,
|
|
|
05485cd |
- co.co_lnotab,
|
|
|
05485cd |
- co.co_cellvars, # this is the trickery
|
|
|
05485cd |
- (),
|
|
|
05485cd |
- )
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- return types.CodeType(
|
|
|
05485cd |
- co.co_argcount,
|
|
|
05485cd |
- co.co_kwonlyargcount,
|
|
|
05485cd |
- co.co_nlocals,
|
|
|
05485cd |
- co.co_stacksize,
|
|
|
05485cd |
- co.co_flags,
|
|
|
05485cd |
- co.co_code,
|
|
|
05485cd |
- co.co_consts,
|
|
|
05485cd |
- co.co_names,
|
|
|
05485cd |
- co.co_varnames,
|
|
|
05485cd |
- co.co_filename,
|
|
|
05485cd |
- co.co_name,
|
|
|
05485cd |
- co.co_firstlineno,
|
|
|
05485cd |
- co.co_lnotab,
|
|
|
05485cd |
- co.co_cellvars, # this is the trickery
|
|
|
05485cd |
- (),
|
|
|
05485cd |
- )
|
|
|
775c617 |
-
|
|
|
775c617 |
-_cell_set_template_code = _make_cell_set_template_code()
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def cell_set(cell, value):
|
|
|
775c617 |
- """Set the value of a closure cell.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- return types.FunctionType(
|
|
|
775c617 |
- _cell_set_template_code,
|
|
|
775c617 |
- {},
|
|
|
775c617 |
- '_cell_set_inner',
|
|
|
775c617 |
- (),
|
|
|
775c617 |
- (cell,),
|
|
|
775c617 |
- )(value)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-# relevant opcodes
|
|
|
775c617 |
-STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
|
|
|
775c617 |
-DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
|
|
|
775c617 |
-LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
|
|
|
775c617 |
-GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
|
|
|
775c617 |
-HAVE_ARGUMENT = dis.HAVE_ARGUMENT
|
|
|
775c617 |
-EXTENDED_ARG = dis.EXTENDED_ARG
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-_BUILTIN_TYPE_NAMES = {}
|
|
|
775c617 |
-for k, v in types.__dict__.items():
|
|
|
775c617 |
- if type(v) is type:
|
|
|
775c617 |
- _BUILTIN_TYPE_NAMES[v] = k
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _builtin_type(name):
|
|
|
775c617 |
- return getattr(types, name)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-if sys.version_info < (3, 4): # pragma: no branch
|
|
|
775c617 |
- def _walk_global_ops(code):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Yield (opcode, argument number) tuples for all
|
|
|
775c617 |
- global-referencing instructions in *code*.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- code = getattr(code, 'co_code', b'')
|
|
|
05485cd |
- if PY2: # pragma: no branch
|
|
|
775c617 |
- code = map(ord, code)
|
|
|
775c617 |
-
|
|
|
775c617 |
- n = len(code)
|
|
|
775c617 |
- i = 0
|
|
|
775c617 |
- extended_arg = 0
|
|
|
775c617 |
- while i < n:
|
|
|
775c617 |
- op = code[i]
|
|
|
775c617 |
- i += 1
|
|
|
775c617 |
- if op >= HAVE_ARGUMENT:
|
|
|
775c617 |
- oparg = code[i] + code[i + 1] * 256 + extended_arg
|
|
|
775c617 |
- extended_arg = 0
|
|
|
775c617 |
- i += 2
|
|
|
775c617 |
- if op == EXTENDED_ARG:
|
|
|
775c617 |
- extended_arg = oparg * 65536
|
|
|
775c617 |
- if op in GLOBAL_OPS:
|
|
|
775c617 |
- yield op, oparg
|
|
|
775c617 |
-
|
|
|
775c617 |
-else:
|
|
|
775c617 |
- def _walk_global_ops(code):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Yield (opcode, argument number) tuples for all
|
|
|
775c617 |
- global-referencing instructions in *code*.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- for instr in dis.get_instructions(code):
|
|
|
775c617 |
- op = instr.opcode
|
|
|
775c617 |
- if op in GLOBAL_OPS:
|
|
|
775c617 |
- yield op, instr.arg
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
05485cd |
-def _extract_class_dict(cls):
|
|
|
05485cd |
- """Retrieve a copy of the dict of a class without the inherited methods"""
|
|
|
05485cd |
- clsdict = dict(cls.__dict__) # copy dict proxy to a dict
|
|
|
05485cd |
- if len(cls.__bases__) == 1:
|
|
|
05485cd |
- inherited_dict = cls.__bases__[0].__dict__
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- inherited_dict = {}
|
|
|
05485cd |
- for base in reversed(cls.__bases__):
|
|
|
05485cd |
- inherited_dict.update(base.__dict__)
|
|
|
05485cd |
- to_remove = []
|
|
|
05485cd |
- for name, value in clsdict.items():
|
|
|
05485cd |
- try:
|
|
|
05485cd |
- base_value = inherited_dict[name]
|
|
|
05485cd |
- if value is base_value:
|
|
|
05485cd |
- to_remove.append(name)
|
|
|
05485cd |
- except KeyError:
|
|
|
05485cd |
- pass
|
|
|
05485cd |
- for name in to_remove:
|
|
|
05485cd |
- clsdict.pop(name)
|
|
|
05485cd |
- return clsdict
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
775c617 |
-class CloudPickler(Pickler):
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch = Pickler.dispatch.copy()
|
|
|
775c617 |
-
|
|
|
775c617 |
- def __init__(self, file, protocol=None):
|
|
|
775c617 |
- if protocol is None:
|
|
|
775c617 |
- protocol = DEFAULT_PROTOCOL
|
|
|
775c617 |
- Pickler.__init__(self, file, protocol=protocol)
|
|
|
775c617 |
- # map ids to dictionary. used to ensure that functions can share global env
|
|
|
775c617 |
- self.globals_ref = {}
|
|
|
775c617 |
-
|
|
|
775c617 |
- def dump(self, obj):
|
|
|
775c617 |
- self.inject_addons()
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- return Pickler.dump(self, obj)
|
|
|
775c617 |
- except RuntimeError as e:
|
|
|
775c617 |
- if 'recursion' in e.args[0]:
|
|
|
775c617 |
- msg = """Could not pickle object as excessively deep recursion required."""
|
|
|
775c617 |
- raise pickle.PicklingError(msg)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- raise
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_memoryview(self, obj):
|
|
|
775c617 |
- self.save(obj.tobytes())
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[memoryview] = save_memoryview
|
|
|
775c617 |
-
|
|
|
05485cd |
- if PY2: # pragma: no branch
|
|
|
775c617 |
- def save_buffer(self, obj):
|
|
|
775c617 |
- self.save(str(obj))
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[buffer] = save_buffer # noqa: F821 'buffer' was removed in Python 3
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_module(self, obj):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Save a module as an import
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if _is_dynamic(obj):
|
|
|
775c617 |
- self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
|
|
|
775c617 |
- obj=obj)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- self.save_reduce(subimport, (obj.__name__,), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[types.ModuleType] = save_module
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_codeobject(self, obj):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Save a code object
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if PY3: # pragma: no branch
|
|
|
05485cd |
- if hasattr(obj, "co_posonlyargcount"): # pragma: no branch
|
|
|
05485cd |
- args = (
|
|
|
05485cd |
- obj.co_argcount, obj.co_posonlyargcount,
|
|
|
05485cd |
- obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
|
05485cd |
- obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
|
05485cd |
- obj.co_varnames, obj.co_filename, obj.co_name,
|
|
|
05485cd |
- obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
|
|
|
05485cd |
- obj.co_cellvars
|
|
|
05485cd |
- )
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- args = (
|
|
|
05485cd |
- obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
|
|
|
05485cd |
- obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
|
|
|
05485cd |
- obj.co_names, obj.co_varnames, obj.co_filename,
|
|
|
05485cd |
- obj.co_name, obj.co_firstlineno, obj.co_lnotab,
|
|
|
05485cd |
- obj.co_freevars, obj.co_cellvars
|
|
|
05485cd |
- )
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- args = (
|
|
|
775c617 |
- obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,
|
|
|
775c617 |
- obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name,
|
|
|
775c617 |
- obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars
|
|
|
775c617 |
- )
|
|
|
775c617 |
- self.save_reduce(types.CodeType, args, obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[types.CodeType] = save_codeobject
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_function(self, obj, name=None):
|
|
|
775c617 |
- """ Registered with the dispatch to handle all function types.
|
|
|
775c617 |
-
|
|
|
775c617 |
- Determines what kind of function obj is (e.g. lambda, defined at
|
|
|
775c617 |
- interactive prompt, etc) and handles the pickling appropriately.
|
|
|
775c617 |
- """
|
|
|
05485cd |
- if _is_global(obj, name=name):
|
|
|
05485cd |
- return Pickler.save_global(self, obj, name=name)
|
|
|
05485cd |
- elif PYPY and isinstance(obj.__code__, builtin_code_type):
|
|
|
05485cd |
- return self.save_pypy_builtin_func(obj)
|
|
|
775c617 |
- else:
|
|
|
05485cd |
- return self.save_function_tuple(obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[types.FunctionType] = save_function
|
|
|
775c617 |
-
|
|
|
05485cd |
- def save_pypy_builtin_func(self, obj):
|
|
|
05485cd |
- """Save pypy equivalent of builtin functions.
|
|
|
05485cd |
-
|
|
|
05485cd |
- PyPy does not have the concept of builtin-functions. Instead,
|
|
|
05485cd |
- builtin-functions are simple function instances, but with a
|
|
|
05485cd |
- builtin-code attribute.
|
|
|
05485cd |
- Most of the time, builtin functions should be pickled by attribute. But
|
|
|
05485cd |
- PyPy has flaky support for __qualname__, so some builtin functions such
|
|
|
05485cd |
- as float.__new__ will be classified as dynamic. For this reason only,
|
|
|
05485cd |
- we created this special routine. Because builtin-functions are not
|
|
|
05485cd |
- expected to have closure or globals, there is no additional hack
|
|
|
05485cd |
- (compared the one already implemented in pickle) to protect ourselves
|
|
|
05485cd |
- from reference cycles. A simple (reconstructor, newargs, obj.__dict__)
|
|
|
05485cd |
- tuple is save_reduced.
|
|
|
05485cd |
-
|
|
|
05485cd |
- Note also that PyPy improved their support for __qualname__ in v3.6, so
|
|
|
05485cd |
- this routing should be removed when cloudpickle supports only PyPy 3.6
|
|
|
05485cd |
- and later.
|
|
|
775c617 |
- """
|
|
|
05485cd |
- rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
|
|
|
05485cd |
- obj.__defaults__, obj.__closure__),
|
|
|
05485cd |
- obj.__dict__)
|
|
|
05485cd |
- self.save_reduce(*rv, obj=obj)
|
|
|
775c617 |
-
|
|
|
05485cd |
- def _save_dynamic_enum(self, obj, clsdict):
|
|
|
05485cd |
- """Special handling for dynamic Enum subclasses
|
|
|
775c617 |
-
|
|
|
05485cd |
- Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the
|
|
|
05485cd |
- EnumMeta metaclass has complex initialization that makes the Enum
|
|
|
05485cd |
- subclasses hold references to their own instances.
|
|
|
05485cd |
- """
|
|
|
05485cd |
- members = dict((e.name, e.value) for e in obj)
|
|
|
775c617 |
-
|
|
|
05485cd |
- # Python 2.7 with enum34 can have no qualname:
|
|
|
05485cd |
- qualname = getattr(obj, "__qualname__", None)
|
|
|
775c617 |
-
|
|
|
05485cd |
- self.save_reduce(_make_skeleton_enum,
|
|
|
05485cd |
- (obj.__bases__, obj.__name__, qualname, members,
|
|
|
05485cd |
- obj.__module__, _ensure_tracking(obj), None),
|
|
|
05485cd |
- obj=obj)
|
|
|
775c617 |
-
|
|
|
05485cd |
- # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
|
|
|
05485cd |
- # Those attributes are already handled by the metaclass.
|
|
|
05485cd |
- for attrname in ["_generate_next_value_", "_member_names_",
|
|
|
05485cd |
- "_member_map_", "_member_type_",
|
|
|
05485cd |
- "_value2member_map_"]:
|
|
|
05485cd |
- clsdict.pop(attrname, None)
|
|
|
05485cd |
- for member in members:
|
|
|
05485cd |
- clsdict.pop(member)
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_dynamic_class(self, obj):
|
|
|
05485cd |
- """Save a class that can't be stored as module global.
|
|
|
775c617 |
-
|
|
|
775c617 |
- This method is used to serialize classes that are defined inside
|
|
|
775c617 |
- functions, or that otherwise can't be serialized as attribute lookups
|
|
|
775c617 |
- from global modules.
|
|
|
775c617 |
- """
|
|
|
05485cd |
- clsdict = _extract_class_dict(obj)
|
|
|
775c617 |
- clsdict.pop('__weakref__', None)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # For ABCMeta in python3.7+, remove _abc_impl as it is not picklable.
|
|
|
775c617 |
- # This is a fix which breaks the cache but this only makes the first
|
|
|
775c617 |
- # calls to issubclass slower.
|
|
|
775c617 |
- if "_abc_impl" in clsdict:
|
|
|
775c617 |
- import abc
|
|
|
775c617 |
- (registry, _, _, _) = abc._get_dump(obj)
|
|
|
775c617 |
- clsdict["_abc_impl"] = [subclass_weakref()
|
|
|
775c617 |
- for subclass_weakref in registry]
|
|
|
775c617 |
-
|
|
|
775c617 |
- # On PyPy, __doc__ is a readonly attribute, so we need to include it in
|
|
|
775c617 |
- # the initial skeleton class. This is safe because we know that the
|
|
|
775c617 |
- # doc can't participate in a cycle with the original class.
|
|
|
775c617 |
- type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
|
|
|
775c617 |
-
|
|
|
775c617 |
- if hasattr(obj, "__slots__"):
|
|
|
775c617 |
- type_kwargs['__slots__'] = obj.__slots__
|
|
|
775c617 |
- # pickle string length optimization: member descriptors of obj are
|
|
|
775c617 |
- # created automatically from obj's __slots__ attribute, no need to
|
|
|
775c617 |
- # save them in obj's state
|
|
|
775c617 |
- if isinstance(obj.__slots__, string_types):
|
|
|
775c617 |
- clsdict.pop(obj.__slots__)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- for k in obj.__slots__:
|
|
|
775c617 |
- clsdict.pop(k, None)
|
|
|
775c617 |
-
|
|
|
05485cd |
- # If type overrides __dict__ as a property, include it in the type
|
|
|
05485cd |
- # kwargs. In Python 2, we can't set this attribute after construction.
|
|
|
775c617 |
- __dict__ = clsdict.pop('__dict__', None)
|
|
|
775c617 |
- if isinstance(__dict__, property):
|
|
|
775c617 |
- type_kwargs['__dict__'] = __dict__
|
|
|
775c617 |
-
|
|
|
775c617 |
- save = self.save
|
|
|
775c617 |
- write = self.write
|
|
|
775c617 |
-
|
|
|
775c617 |
- # We write pickle instructions explicitly here to handle the
|
|
|
775c617 |
- # possibility that the type object participates in a cycle with its own
|
|
|
775c617 |
- # __dict__. We first write an empty "skeleton" version of the class and
|
|
|
775c617 |
- # memoize it before writing the class' __dict__ itself. We then write
|
|
|
775c617 |
- # instructions to "rehydrate" the skeleton class by restoring the
|
|
|
775c617 |
- # attributes from the __dict__.
|
|
|
775c617 |
- #
|
|
|
775c617 |
- # A type can appear in a cycle with its __dict__ if an instance of the
|
|
|
775c617 |
- # type appears in the type's __dict__ (which happens for the stdlib
|
|
|
775c617 |
- # Enum class), or if the type defines methods that close over the name
|
|
|
775c617 |
- # of the type, (which is common for Python 2-style super() calls).
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Push the rehydration function.
|
|
|
775c617 |
- save(_rehydrate_skeleton_class)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Mark the start of the args tuple for the rehydration function.
|
|
|
775c617 |
- write(pickle.MARK)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Create and memoize an skeleton class with obj's name and bases.
|
|
|
05485cd |
- if Enum is not None and issubclass(obj, Enum):
|
|
|
05485cd |
- # Special handling of Enum subclasses
|
|
|
05485cd |
- self._save_dynamic_enum(obj, clsdict)
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- # "Regular" class definition:
|
|
|
05485cd |
- tp = type(obj)
|
|
|
05485cd |
- self.save_reduce(_make_skeleton_class,
|
|
|
05485cd |
- (tp, obj.__name__, obj.__bases__, type_kwargs,
|
|
|
05485cd |
- _ensure_tracking(obj), None),
|
|
|
05485cd |
- obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Now save the rest of obj's __dict__. Any references to obj
|
|
|
775c617 |
- # encountered while saving will point to the skeleton class.
|
|
|
775c617 |
- save(clsdict)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Write a tuple of (skeleton_class, clsdict).
|
|
|
775c617 |
- write(pickle.TUPLE)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Call _rehydrate_skeleton_class(skeleton_class, clsdict)
|
|
|
775c617 |
- write(pickle.REDUCE)
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_function_tuple(self, func):
|
|
|
775c617 |
- """ Pickles an actual func object.
|
|
|
775c617 |
-
|
|
|
775c617 |
- A func comprises: code, globals, defaults, closure, and dict. We
|
|
|
775c617 |
- extract and save these, injecting reducing functions at certain points
|
|
|
775c617 |
- to recreate the func object. Keep in mind that some of these pieces
|
|
|
775c617 |
- can contain a ref to the func itself. Thus, a naive save on these
|
|
|
775c617 |
- pieces could trigger an infinite loop of save's. To get around that,
|
|
|
775c617 |
- we first create a skeleton func object using just the code (this is
|
|
|
775c617 |
- safe, since this won't contain a ref to the func), and memoize it as
|
|
|
775c617 |
- soon as it's created. The other stuff can then be filled in later.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if is_tornado_coroutine(func):
|
|
|
775c617 |
- self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,),
|
|
|
775c617 |
- obj=func)
|
|
|
775c617 |
- return
|
|
|
775c617 |
-
|
|
|
775c617 |
- save = self.save
|
|
|
775c617 |
- write = self.write
|
|
|
775c617 |
-
|
|
|
775c617 |
- code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)
|
|
|
775c617 |
-
|
|
|
775c617 |
- save(_fill_function) # skeleton function updater
|
|
|
775c617 |
- write(pickle.MARK) # beginning of tuple that _fill_function expects
|
|
|
775c617 |
-
|
|
|
05485cd |
- # Extract currently-imported submodules used by func. Storing these
|
|
|
05485cd |
- # modules in a smoke _cloudpickle_subimports attribute of the object's
|
|
|
05485cd |
- # state will trigger the side effect of importing these modules at
|
|
|
05485cd |
- # unpickling time (which is necessary for func to work correctly once
|
|
|
05485cd |
- # depickled)
|
|
|
05485cd |
- submodules = _find_imported_submodules(
|
|
|
775c617 |
- code,
|
|
|
775c617 |
- itertools.chain(f_globals.values(), closure_values or ()),
|
|
|
775c617 |
- )
|
|
|
775c617 |
-
|
|
|
775c617 |
- # create a skeleton function object and memoize it
|
|
|
775c617 |
- save(_make_skel_func)
|
|
|
775c617 |
- save((
|
|
|
775c617 |
- code,
|
|
|
775c617 |
- len(closure_values) if closure_values is not None else -1,
|
|
|
775c617 |
- base_globals,
|
|
|
775c617 |
- ))
|
|
|
775c617 |
- write(pickle.REDUCE)
|
|
|
775c617 |
- self.memoize(func)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # save the rest of the func data needed by _fill_function
|
|
|
775c617 |
- state = {
|
|
|
775c617 |
- 'globals': f_globals,
|
|
|
775c617 |
- 'defaults': defaults,
|
|
|
775c617 |
- 'dict': dct,
|
|
|
775c617 |
- 'closure_values': closure_values,
|
|
|
775c617 |
- 'module': func.__module__,
|
|
|
775c617 |
- 'name': func.__name__,
|
|
|
775c617 |
- 'doc': func.__doc__,
|
|
|
05485cd |
- '_cloudpickle_submodules': submodules
|
|
|
775c617 |
- }
|
|
|
05485cd |
- if hasattr(func, '__annotations__') and sys.version_info >= (3, 4):
|
|
|
775c617 |
- state['annotations'] = func.__annotations__
|
|
|
775c617 |
- if hasattr(func, '__qualname__'):
|
|
|
775c617 |
- state['qualname'] = func.__qualname__
|
|
|
05485cd |
- if hasattr(func, '__kwdefaults__'):
|
|
|
05485cd |
- state['kwdefaults'] = func.__kwdefaults__
|
|
|
775c617 |
- save(state)
|
|
|
775c617 |
- write(pickle.TUPLE)
|
|
|
775c617 |
- write(pickle.REDUCE) # applies _fill_function on the tuple
|
|
|
775c617 |
-
|
|
|
775c617 |
- def extract_func_data(self, func):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Turn the function into a tuple of data necessary to recreate it:
|
|
|
775c617 |
- code, globals, defaults, closure_values, dict
|
|
|
775c617 |
- """
|
|
|
775c617 |
- code = func.__code__
|
|
|
775c617 |
-
|
|
|
775c617 |
- # extract all global ref's
|
|
|
05485cd |
- func_global_refs = _extract_code_globals(code)
|
|
|
775c617 |
-
|
|
|
775c617 |
- # process all variables referenced by global environment
|
|
|
775c617 |
- f_globals = {}
|
|
|
775c617 |
- for var in func_global_refs:
|
|
|
775c617 |
- if var in func.__globals__:
|
|
|
775c617 |
- f_globals[var] = func.__globals__[var]
|
|
|
775c617 |
-
|
|
|
775c617 |
- # defaults requires no processing
|
|
|
775c617 |
- defaults = func.__defaults__
|
|
|
775c617 |
-
|
|
|
775c617 |
- # process closure
|
|
|
775c617 |
- closure = (
|
|
|
775c617 |
- list(map(_get_cell_contents, func.__closure__))
|
|
|
775c617 |
- if func.__closure__ is not None
|
|
|
775c617 |
- else None
|
|
|
775c617 |
- )
|
|
|
775c617 |
-
|
|
|
775c617 |
- # save the dict
|
|
|
775c617 |
- dct = func.__dict__
|
|
|
775c617 |
-
|
|
|
775c617 |
- # base_globals represents the future global namespace of func at
|
|
|
775c617 |
- # unpickling time. Looking it up and storing it in globals_ref allow
|
|
|
775c617 |
- # functions sharing the same globals at pickling time to also
|
|
|
775c617 |
- # share them once unpickled, at one condition: since globals_ref is
|
|
|
775c617 |
- # an attribute of a Cloudpickler instance, and that a new CloudPickler is
|
|
|
775c617 |
- # created each time pickle.dump or pickle.dumps is called, functions
|
|
|
775c617 |
- # also need to be saved within the same invokation of
|
|
|
775c617 |
- # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There
|
|
|
775c617 |
- # is no such limitation when using Cloudpickler.dump, as long as the
|
|
|
775c617 |
- # multiple invokations are bound to the same Cloudpickler.
|
|
|
775c617 |
- base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
|
|
|
775c617 |
-
|
|
|
05485cd |
- if base_globals == {}:
|
|
|
05485cd |
- # Add module attributes used to resolve relative imports
|
|
|
05485cd |
- # instructions inside func.
|
|
|
05485cd |
- for k in ["__package__", "__name__", "__path__", "__file__"]:
|
|
|
05485cd |
- # Some built-in functions/methods such as object.__new__ have
|
|
|
05485cd |
- # their __globals__ set to None in PyPy
|
|
|
05485cd |
- if func.__globals__ is not None and k in func.__globals__:
|
|
|
05485cd |
- base_globals[k] = func.__globals__[k]
|
|
|
775c617 |
-
|
|
|
05485cd |
- return (code, f_globals, defaults, closure, dct, base_globals)
|
|
|
775c617 |
-
|
|
|
05485cd |
- if not PY3: # pragma: no branch
|
|
|
05485cd |
- # Python3 comes with native reducers that allow builtin functions and
|
|
|
05485cd |
- # methods pickling as module/class attributes. The following method
|
|
|
05485cd |
- # extends this for python2.
|
|
|
05485cd |
- # Please note that currently, neither pickle nor cloudpickle support
|
|
|
05485cd |
- # dynamically created builtin functions/method pickling.
|
|
|
05485cd |
- def save_builtin_function_or_method(self, obj):
|
|
|
05485cd |
- is_bound = getattr(obj, '__self__', None) is not None
|
|
|
05485cd |
- if is_bound:
|
|
|
05485cd |
- # obj is a bound builtin method.
|
|
|
05485cd |
- rv = (getattr, (obj.__self__, obj.__name__))
|
|
|
05485cd |
- return self.save_reduce(obj=obj, *rv)
|
|
|
05485cd |
-
|
|
|
05485cd |
- is_unbound = hasattr(obj, '__objclass__')
|
|
|
05485cd |
- if is_unbound:
|
|
|
05485cd |
- # obj is an unbound builtin method (accessed from its class)
|
|
|
05485cd |
- rv = (getattr, (obj.__objclass__, obj.__name__))
|
|
|
05485cd |
- return self.save_reduce(obj=obj, *rv)
|
|
|
05485cd |
-
|
|
|
05485cd |
- # Otherwise, obj is not a method, but a function. Fallback to
|
|
|
05485cd |
- # default pickling by attribute.
|
|
|
05485cd |
- return Pickler.save_global(self, obj)
|
|
|
05485cd |
-
|
|
|
05485cd |
- dispatch[types.BuiltinFunctionType] = save_builtin_function_or_method
|
|
|
05485cd |
-
|
|
|
05485cd |
- # A comprehensive summary of the various kinds of builtin methods can
|
|
|
05485cd |
- # be found in PEP 579: https://www.python.org/dev/peps/pep-0579/
|
|
|
05485cd |
- classmethod_descriptor_type = type(float.__dict__['fromhex'])
|
|
|
05485cd |
- wrapper_descriptor_type = type(float.__repr__)
|
|
|
05485cd |
- method_wrapper_type = type(1.5.__repr__)
|
|
|
05485cd |
-
|
|
|
05485cd |
- dispatch[classmethod_descriptor_type] = save_builtin_function_or_method
|
|
|
05485cd |
- dispatch[wrapper_descriptor_type] = save_builtin_function_or_method
|
|
|
05485cd |
- dispatch[method_wrapper_type] = save_builtin_function_or_method
|
|
|
05485cd |
-
|
|
|
05485cd |
- if sys.version_info[:2] < (3, 4):
|
|
|
05485cd |
- method_descriptor = type(str.upper)
|
|
|
05485cd |
- dispatch[method_descriptor] = save_builtin_function_or_method
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_global(self, obj, name=None, pack=struct.pack):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Save a "global".
|
|
|
775c617 |
-
|
|
|
775c617 |
- The name of this method is somewhat misleading: all types get
|
|
|
775c617 |
- dispatched here.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if obj is type(None):
|
|
|
775c617 |
- return self.save_reduce(type, (None,), obj=obj)
|
|
|
775c617 |
- elif obj is type(Ellipsis):
|
|
|
775c617 |
- return self.save_reduce(type, (Ellipsis,), obj=obj)
|
|
|
775c617 |
- elif obj is type(NotImplemented):
|
|
|
775c617 |
- return self.save_reduce(type, (NotImplemented,), obj=obj)
|
|
|
05485cd |
- elif obj in _BUILTIN_TYPE_NAMES:
|
|
|
05485cd |
- return self.save_reduce(
|
|
|
05485cd |
- _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
|
|
|
05485cd |
- elif name is not None:
|
|
|
05485cd |
- Pickler.save_global(self, obj, name=name)
|
|
|
05485cd |
- elif not _is_global(obj, name=name):
|
|
|
05485cd |
- self.save_dynamic_class(obj)
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- Pickler.save_global(self, obj, name=name)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[type] = save_global
|
|
|
775c617 |
- dispatch[types.ClassType] = save_global
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_instancemethod(self, obj):
|
|
|
775c617 |
- # Memoization rarely is ever useful due to python bounding
|
|
|
775c617 |
- if obj.__self__ is None:
|
|
|
775c617 |
- self.save_reduce(getattr, (obj.im_class, obj.__name__))
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- if PY3: # pragma: no branch
|
|
|
775c617 |
- self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__),
|
|
|
775c617 |
- obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[types.MethodType] = save_instancemethod
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_inst(self, obj):
|
|
|
775c617 |
- """Inner logic to save instance. Based off pickle.save_inst"""
|
|
|
775c617 |
- cls = obj.__class__
|
|
|
775c617 |
-
|
|
|
775c617 |
- # Try the dispatch table (pickle module doesn't do it)
|
|
|
775c617 |
- f = self.dispatch.get(cls)
|
|
|
775c617 |
- if f:
|
|
|
775c617 |
- f(self, obj) # Call unbound method with explicit self
|
|
|
775c617 |
- return
|
|
|
775c617 |
-
|
|
|
775c617 |
- memo = self.memo
|
|
|
775c617 |
- write = self.write
|
|
|
775c617 |
- save = self.save
|
|
|
775c617 |
-
|
|
|
775c617 |
- if hasattr(obj, '__getinitargs__'):
|
|
|
775c617 |
- args = obj.__getinitargs__()
|
|
|
775c617 |
- len(args) # XXX Assert it's a sequence
|
|
|
775c617 |
- pickle._keep_alive(args, memo)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- args = ()
|
|
|
775c617 |
-
|
|
|
775c617 |
- write(pickle.MARK)
|
|
|
775c617 |
-
|
|
|
775c617 |
- if self.bin:
|
|
|
775c617 |
- save(cls)
|
|
|
775c617 |
- for arg in args:
|
|
|
775c617 |
- save(arg)
|
|
|
775c617 |
- write(pickle.OBJ)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- for arg in args:
|
|
|
775c617 |
- save(arg)
|
|
|
775c617 |
- write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n')
|
|
|
775c617 |
-
|
|
|
775c617 |
- self.memoize(obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- getstate = obj.__getstate__
|
|
|
775c617 |
- except AttributeError:
|
|
|
775c617 |
- stuff = obj.__dict__
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- stuff = getstate()
|
|
|
775c617 |
- pickle._keep_alive(stuff, memo)
|
|
|
775c617 |
- save(stuff)
|
|
|
775c617 |
- write(pickle.BUILD)
|
|
|
775c617 |
-
|
|
|
05485cd |
- if PY2: # pragma: no branch
|
|
|
775c617 |
- dispatch[types.InstanceType] = save_inst
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_property(self, obj):
|
|
|
775c617 |
- # properties not correctly saved in python
|
|
|
775c617 |
- self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[property] = save_property
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_classmethod(self, obj):
|
|
|
775c617 |
- orig_func = obj.__func__
|
|
|
775c617 |
- self.save_reduce(type(obj), (orig_func,), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[classmethod] = save_classmethod
|
|
|
775c617 |
- dispatch[staticmethod] = save_classmethod
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_itemgetter(self, obj):
|
|
|
775c617 |
- """itemgetter serializer (needed for namedtuple support)"""
|
|
|
775c617 |
- class Dummy:
|
|
|
775c617 |
- def __getitem__(self, item):
|
|
|
775c617 |
- return item
|
|
|
775c617 |
- items = obj(Dummy())
|
|
|
775c617 |
- if not isinstance(items, tuple):
|
|
|
775c617 |
- items = (items,)
|
|
|
775c617 |
- return self.save_reduce(operator.itemgetter, items)
|
|
|
775c617 |
-
|
|
|
775c617 |
- if type(operator.itemgetter) is type:
|
|
|
775c617 |
- dispatch[operator.itemgetter] = save_itemgetter
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_attrgetter(self, obj):
|
|
|
775c617 |
- """attrgetter serializer"""
|
|
|
775c617 |
- class Dummy(object):
|
|
|
775c617 |
- def __init__(self, attrs, index=None):
|
|
|
775c617 |
- self.attrs = attrs
|
|
|
775c617 |
- self.index = index
|
|
|
775c617 |
- def __getattribute__(self, item):
|
|
|
775c617 |
- attrs = object.__getattribute__(self, "attrs")
|
|
|
775c617 |
- index = object.__getattribute__(self, "index")
|
|
|
775c617 |
- if index is None:
|
|
|
775c617 |
- index = len(attrs)
|
|
|
775c617 |
- attrs.append(item)
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- attrs[index] = ".".join([attrs[index], item])
|
|
|
775c617 |
- return type(self)(attrs, index)
|
|
|
775c617 |
- attrs = []
|
|
|
775c617 |
- obj(Dummy(attrs))
|
|
|
775c617 |
- return self.save_reduce(operator.attrgetter, tuple(attrs))
|
|
|
775c617 |
-
|
|
|
775c617 |
- if type(operator.attrgetter) is type:
|
|
|
775c617 |
- dispatch[operator.attrgetter] = save_attrgetter
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_file(self, obj):
|
|
|
775c617 |
- """Save a file"""
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- import StringIO as pystringIO # we can't use cStringIO as it lacks the name attribute
|
|
|
775c617 |
- except ImportError:
|
|
|
775c617 |
- import io as pystringIO
|
|
|
775c617 |
-
|
|
|
775c617 |
- if not hasattr(obj, 'name') or not hasattr(obj, 'mode'):
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
|
|
|
775c617 |
- if obj is sys.stdout:
|
|
|
775c617 |
- return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
|
|
|
775c617 |
- if obj is sys.stderr:
|
|
|
775c617 |
- return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
|
|
|
775c617 |
- if obj is sys.stdin:
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle standard input")
|
|
|
775c617 |
- if obj.closed:
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle closed files")
|
|
|
775c617 |
- if hasattr(obj, 'isatty') and obj.isatty():
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle files that map to tty objects")
|
|
|
775c617 |
- if 'r' not in obj.mode and '+' not in obj.mode:
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)
|
|
|
775c617 |
-
|
|
|
775c617 |
- name = obj.name
|
|
|
775c617 |
-
|
|
|
775c617 |
- retval = pystringIO.StringIO()
|
|
|
775c617 |
-
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- # Read the whole file
|
|
|
775c617 |
- curloc = obj.tell()
|
|
|
775c617 |
- obj.seek(0)
|
|
|
775c617 |
- contents = obj.read()
|
|
|
775c617 |
- obj.seek(curloc)
|
|
|
775c617 |
- except IOError:
|
|
|
775c617 |
- raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
|
|
|
775c617 |
- retval.write(contents)
|
|
|
775c617 |
- retval.seek(curloc)
|
|
|
775c617 |
-
|
|
|
775c617 |
- retval.name = name
|
|
|
775c617 |
- self.save(retval)
|
|
|
775c617 |
- self.memoize(obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_ellipsis(self, obj):
|
|
|
775c617 |
- self.save_reduce(_gen_ellipsis, ())
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_not_implemented(self, obj):
|
|
|
775c617 |
- self.save_reduce(_gen_not_implemented, ())
|
|
|
775c617 |
-
|
|
|
775c617 |
- try: # Python 2
|
|
|
775c617 |
- dispatch[file] = save_file
|
|
|
775c617 |
- except NameError: # Python 3 # pragma: no branch
|
|
|
775c617 |
- dispatch[io.TextIOWrapper] = save_file
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[type(Ellipsis)] = save_ellipsis
|
|
|
775c617 |
- dispatch[type(NotImplemented)] = save_not_implemented
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_weakset(self, obj):
|
|
|
775c617 |
- self.save_reduce(weakref.WeakSet, (list(obj),))
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[weakref.WeakSet] = save_weakset
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_logger(self, obj):
|
|
|
775c617 |
- self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[logging.Logger] = save_logger
|
|
|
775c617 |
-
|
|
|
775c617 |
- def save_root_logger(self, obj):
|
|
|
775c617 |
- self.save_reduce(logging.getLogger, (), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[logging.RootLogger] = save_root_logger
|
|
|
775c617 |
-
|
|
|
775c617 |
- if hasattr(types, "MappingProxyType"): # pragma: no branch
|
|
|
775c617 |
- def save_mappingproxy(self, obj):
|
|
|
775c617 |
- self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
- dispatch[types.MappingProxyType] = save_mappingproxy
|
|
|
775c617 |
-
|
|
|
775c617 |
- """Special functions for Add-on libraries"""
|
|
|
775c617 |
- def inject_addons(self):
|
|
|
775c617 |
- """Plug in system. Register additional pickling functions if modules already loaded"""
|
|
|
775c617 |
- pass
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-# Tornado support
|
|
|
775c617 |
-
|
|
|
775c617 |
-def is_tornado_coroutine(func):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Return whether *func* is a Tornado coroutine function.
|
|
|
775c617 |
- Running coroutines are not supported.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if 'tornado.gen' not in sys.modules:
|
|
|
775c617 |
- return False
|
|
|
775c617 |
- gen = sys.modules['tornado.gen']
|
|
|
775c617 |
- if not hasattr(gen, "is_coroutine_function"):
|
|
|
775c617 |
- # Tornado version is too old
|
|
|
775c617 |
- return False
|
|
|
775c617 |
- return gen.is_coroutine_function(func)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _rebuild_tornado_coroutine(func):
|
|
|
775c617 |
- from tornado import gen
|
|
|
775c617 |
- return gen.coroutine(func)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-# Shorthands for legacy support
|
|
|
775c617 |
-
|
|
|
775c617 |
-def dump(obj, file, protocol=None):
|
|
|
775c617 |
- """Serialize obj as bytes streamed into file
|
|
|
775c617 |
-
|
|
|
775c617 |
- protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
|
|
|
775c617 |
- pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
|
|
|
775c617 |
- between processes running the same Python version.
|
|
|
775c617 |
-
|
|
|
775c617 |
- Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
|
|
|
775c617 |
- compatibility with older versions of Python.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- CloudPickler(file, protocol=protocol).dump(obj)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def dumps(obj, protocol=None):
|
|
|
775c617 |
- """Serialize obj as a string of bytes allocated in memory
|
|
|
775c617 |
-
|
|
|
775c617 |
- protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
|
|
|
775c617 |
- pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
|
|
|
775c617 |
- between processes running the same Python version.
|
|
|
775c617 |
-
|
|
|
775c617 |
- Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
|
|
|
775c617 |
- compatibility with older versions of Python.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- file = StringIO()
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- cp = CloudPickler(file, protocol=protocol)
|
|
|
775c617 |
- cp.dump(obj)
|
|
|
775c617 |
- return file.getvalue()
|
|
|
775c617 |
- finally:
|
|
|
775c617 |
- file.close()
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-# including pickles unloading functions in this namespace
|
|
|
775c617 |
-load = pickle.load
|
|
|
775c617 |
-loads = pickle.loads
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-# hack for __import__ not working as desired
|
|
|
775c617 |
-def subimport(name):
|
|
|
775c617 |
- __import__(name)
|
|
|
775c617 |
- return sys.modules[name]
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def dynamic_subimport(name, vars):
|
|
|
775c617 |
- mod = types.ModuleType(name)
|
|
|
775c617 |
- mod.__dict__.update(vars)
|
|
|
775c617 |
- return mod
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _gen_ellipsis():
|
|
|
775c617 |
- return Ellipsis
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _gen_not_implemented():
|
|
|
775c617 |
- return NotImplemented
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _get_cell_contents(cell):
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- return cell.cell_contents
|
|
|
775c617 |
- except ValueError:
|
|
|
775c617 |
- # sentinel used by ``_fill_function`` which will leave the cell empty
|
|
|
775c617 |
- return _empty_cell_value
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def instance(cls):
|
|
|
775c617 |
- """Create a new instance of a class.
|
|
|
775c617 |
-
|
|
|
775c617 |
- Parameters
|
|
|
775c617 |
- ----------
|
|
|
775c617 |
- cls : type
|
|
|
775c617 |
- The class to create an instance of.
|
|
|
775c617 |
-
|
|
|
775c617 |
- Returns
|
|
|
775c617 |
- -------
|
|
|
775c617 |
- instance : cls
|
|
|
775c617 |
- A new instance of ``cls``.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- return cls()
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-@instance
|
|
|
775c617 |
-class _empty_cell_value(object):
|
|
|
775c617 |
- """sentinel for empty closures
|
|
|
775c617 |
- """
|
|
|
775c617 |
- @classmethod
|
|
|
775c617 |
- def __reduce__(cls):
|
|
|
775c617 |
- return cls.__name__
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _fill_function(*args):
|
|
|
775c617 |
- """Fills in the rest of function data into the skeleton function object
|
|
|
775c617 |
-
|
|
|
775c617 |
- The skeleton itself is create by _make_skel_func().
|
|
|
775c617 |
- """
|
|
|
775c617 |
- if len(args) == 2:
|
|
|
775c617 |
- func = args[0]
|
|
|
775c617 |
- state = args[1]
|
|
|
775c617 |
- elif len(args) == 5:
|
|
|
775c617 |
- # Backwards compat for cloudpickle v0.4.0, after which the `module`
|
|
|
775c617 |
- # argument was introduced
|
|
|
775c617 |
- func = args[0]
|
|
|
775c617 |
- keys = ['globals', 'defaults', 'dict', 'closure_values']
|
|
|
775c617 |
- state = dict(zip(keys, args[1:]))
|
|
|
775c617 |
- elif len(args) == 6:
|
|
|
775c617 |
- # Backwards compat for cloudpickle v0.4.1, after which the function
|
|
|
775c617 |
- # state was passed as a dict to the _fill_function it-self.
|
|
|
775c617 |
- func = args[0]
|
|
|
775c617 |
- keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
|
|
|
775c617 |
- state = dict(zip(keys, args[1:]))
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
|
|
|
775c617 |
-
|
|
|
775c617 |
- # - At pickling time, any dynamic global variable used by func is
|
|
|
775c617 |
- # serialized by value (in state['globals']).
|
|
|
775c617 |
- # - At unpickling time, func's __globals__ attribute is initialized by
|
|
|
775c617 |
- # first retrieving an empty isolated namespace that will be shared
|
|
|
775c617 |
- # with other functions pickled from the same original module
|
|
|
775c617 |
- # by the same CloudPickler instance and then updated with the
|
|
|
775c617 |
- # content of state['globals'] to populate the shared isolated
|
|
|
775c617 |
- # namespace with all the global variables that are specifically
|
|
|
775c617 |
- # referenced for this function.
|
|
|
775c617 |
- func.__globals__.update(state['globals'])
|
|
|
775c617 |
-
|
|
|
775c617 |
- func.__defaults__ = state['defaults']
|
|
|
775c617 |
- func.__dict__ = state['dict']
|
|
|
775c617 |
- if 'annotations' in state:
|
|
|
775c617 |
- func.__annotations__ = state['annotations']
|
|
|
775c617 |
- if 'doc' in state:
|
|
|
775c617 |
- func.__doc__ = state['doc']
|
|
|
775c617 |
- if 'name' in state:
|
|
|
775c617 |
- func.__name__ = state['name']
|
|
|
775c617 |
- if 'module' in state:
|
|
|
775c617 |
- func.__module__ = state['module']
|
|
|
775c617 |
- if 'qualname' in state:
|
|
|
775c617 |
- func.__qualname__ = state['qualname']
|
|
|
05485cd |
- if 'kwdefaults' in state:
|
|
|
05485cd |
- func.__kwdefaults__ = state['kwdefaults']
|
|
|
05485cd |
- # _cloudpickle_subimports is a set of submodules that must be loaded for
|
|
|
05485cd |
- # the pickled function to work correctly at unpickling time. Now that these
|
|
|
05485cd |
- # submodules are depickled (hence imported), they can be removed from the
|
|
|
05485cd |
- # object's state (the object state only served as a reference holder to
|
|
|
05485cd |
- # these submodules)
|
|
|
05485cd |
- if '_cloudpickle_submodules' in state:
|
|
|
05485cd |
- state.pop('_cloudpickle_submodules')
|
|
|
775c617 |
-
|
|
|
775c617 |
- cells = func.__closure__
|
|
|
775c617 |
- if cells is not None:
|
|
|
775c617 |
- for cell, value in zip(cells, state['closure_values']):
|
|
|
775c617 |
- if value is not _empty_cell_value:
|
|
|
775c617 |
- cell_set(cell, value)
|
|
|
775c617 |
-
|
|
|
775c617 |
- return func
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _make_empty_cell():
|
|
|
775c617 |
- if False:
|
|
|
775c617 |
- # trick the compiler into creating an empty cell in our lambda
|
|
|
775c617 |
- cell = None
|
|
|
775c617 |
- raise AssertionError('this route should not be executed')
|
|
|
775c617 |
-
|
|
|
775c617 |
- return (lambda: cell).__closure__[0]
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
775c617 |
-def _make_skel_func(code, cell_count, base_globals=None):
|
|
|
775c617 |
- """ Creates a skeleton function object that contains just the provided
|
|
|
775c617 |
- code and the correct number of cells in func_closure. All other
|
|
|
775c617 |
- func attributes (e.g. func_globals) are empty.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- # This is backward-compatibility code: for cloudpickle versions between
|
|
|
775c617 |
- # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
|
|
|
775c617 |
- # should now always be a dictionary.
|
|
|
775c617 |
- if base_globals is None or isinstance(base_globals, str):
|
|
|
775c617 |
- base_globals = {}
|
|
|
775c617 |
-
|
|
|
775c617 |
- base_globals['__builtins__'] = __builtins__
|
|
|
775c617 |
-
|
|
|
775c617 |
- closure = (
|
|
|
775c617 |
- tuple(_make_empty_cell() for _ in range(cell_count))
|
|
|
775c617 |
- if cell_count >= 0 else
|
|
|
775c617 |
- None
|
|
|
775c617 |
- )
|
|
|
775c617 |
- return types.FunctionType(code, base_globals, None, None, closure)
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
05485cd |
-def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
|
|
|
05485cd |
- class_tracker_id, extra):
|
|
|
05485cd |
- """Build dynamic class with an empty __dict__ to be filled once memoized
|
|
|
05485cd |
-
|
|
|
05485cd |
- If class_tracker_id is not None, try to lookup an existing class definition
|
|
|
05485cd |
- matching that id. If none is found, track a newly reconstructed class
|
|
|
05485cd |
- definition under that id so that other instances stemming from the same
|
|
|
05485cd |
- class id will also reuse this class definition.
|
|
|
05485cd |
-
|
|
|
05485cd |
- The "extra" variable is meant to be a dict (or None) that can be used for
|
|
|
05485cd |
- forward compatibility shall the need arise.
|
|
|
05485cd |
- """
|
|
|
05485cd |
- skeleton_class = type_constructor(name, bases, type_kwargs)
|
|
|
05485cd |
- return _lookup_class_or_track(class_tracker_id, skeleton_class)
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
775c617 |
-def _rehydrate_skeleton_class(skeleton_class, class_dict):
|
|
|
775c617 |
- """Put attributes from `class_dict` back on `skeleton_class`.
|
|
|
775c617 |
-
|
|
|
775c617 |
- See CloudPickler.save_dynamic_class for more info.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- registry = None
|
|
|
775c617 |
- for attrname, attr in class_dict.items():
|
|
|
775c617 |
- if attrname == "_abc_impl":
|
|
|
775c617 |
- registry = attr
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- setattr(skeleton_class, attrname, attr)
|
|
|
775c617 |
- if registry is not None:
|
|
|
775c617 |
- for subclass in registry:
|
|
|
775c617 |
- skeleton_class.register(subclass)
|
|
|
775c617 |
-
|
|
|
775c617 |
- return skeleton_class
|
|
|
775c617 |
-
|
|
|
775c617 |
-
|
|
|
05485cd |
-def _make_skeleton_enum(bases, name, qualname, members, module,
|
|
|
05485cd |
- class_tracker_id, extra):
|
|
|
05485cd |
- """Build dynamic enum with an empty __dict__ to be filled once memoized
|
|
|
05485cd |
-
|
|
|
05485cd |
- The creation of the enum class is inspired by the code of
|
|
|
05485cd |
- EnumMeta._create_.
|
|
|
05485cd |
-
|
|
|
05485cd |
- If class_tracker_id is not None, try to lookup an existing enum definition
|
|
|
05485cd |
- matching that id. If none is found, track a newly reconstructed enum
|
|
|
05485cd |
- definition under that id so that other instances stemming from the same
|
|
|
05485cd |
- class id will also reuse this enum definition.
|
|
|
05485cd |
-
|
|
|
05485cd |
- The "extra" variable is meant to be a dict (or None) that can be used for
|
|
|
05485cd |
- forward compatibility shall the need arise.
|
|
|
05485cd |
- """
|
|
|
05485cd |
- # enums always inherit from their base Enum class at the last position in
|
|
|
05485cd |
- # the list of base classes:
|
|
|
05485cd |
- enum_base = bases[-1]
|
|
|
05485cd |
- metacls = enum_base.__class__
|
|
|
05485cd |
- classdict = metacls.__prepare__(name, bases)
|
|
|
05485cd |
-
|
|
|
05485cd |
- for member_name, member_value in members.items():
|
|
|
05485cd |
- classdict[member_name] = member_value
|
|
|
05485cd |
- enum_class = metacls.__new__(metacls, name, bases, classdict)
|
|
|
05485cd |
- enum_class.__module__ = module
|
|
|
05485cd |
-
|
|
|
05485cd |
- # Python 2.7 compat
|
|
|
05485cd |
- if qualname is not None:
|
|
|
05485cd |
- enum_class.__qualname__ = qualname
|
|
|
05485cd |
-
|
|
|
05485cd |
- return _lookup_class_or_track(class_tracker_id, enum_class)
|
|
|
05485cd |
-
|
|
|
05485cd |
-
|
|
|
775c617 |
-def _is_dynamic(module):
|
|
|
775c617 |
- """
|
|
|
775c617 |
- Return True if the module is special module that cannot be imported by its
|
|
|
775c617 |
- name.
|
|
|
775c617 |
- """
|
|
|
775c617 |
- # Quick check: module that have __file__ attribute are not dynamic modules.
|
|
|
775c617 |
- if hasattr(module, '__file__'):
|
|
|
775c617 |
- return False
|
|
|
775c617 |
-
|
|
|
775c617 |
- if hasattr(module, '__spec__'):
|
|
|
05485cd |
- if module.__spec__ is not None:
|
|
|
05485cd |
- return False
|
|
|
05485cd |
-
|
|
|
05485cd |
- # In PyPy, Some built-in modules such as _codecs can have their
|
|
|
05485cd |
- # __spec__ attribute set to None despite being imported. For such
|
|
|
05485cd |
- # modules, the ``_find_spec`` utility of the standard library is used.
|
|
|
05485cd |
- parent_name = module.__name__.rpartition('.')[0]
|
|
|
05485cd |
- if parent_name: # pragma: no cover
|
|
|
05485cd |
- # This code handles the case where an imported package (and not
|
|
|
05485cd |
- # module) remains with __spec__ set to None. It is however untested
|
|
|
05485cd |
- # as no package in the PyPy stdlib has __spec__ set to None after
|
|
|
05485cd |
- # it is imported.
|
|
|
05485cd |
- try:
|
|
|
05485cd |
- parent = sys.modules[parent_name]
|
|
|
05485cd |
- except KeyError:
|
|
|
05485cd |
- msg = "parent {!r} not in sys.modules"
|
|
|
05485cd |
- raise ImportError(msg.format(parent_name))
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- pkgpath = parent.__path__
|
|
|
05485cd |
- else:
|
|
|
05485cd |
- pkgpath = None
|
|
|
05485cd |
- return _find_spec(module.__name__, pkgpath, module) is None
|
|
|
05485cd |
-
|
|
|
775c617 |
- else:
|
|
|
775c617 |
- # Backward compat for Python 2
|
|
|
775c617 |
- import imp
|
|
|
775c617 |
- try:
|
|
|
775c617 |
- path = None
|
|
|
775c617 |
- for part in module.__name__.split('.'):
|
|
|
775c617 |
- if path is not None:
|
|
|
775c617 |
- path = [path]
|
|
|
775c617 |
- f, path, description = imp.find_module(part, path)
|
|
|
775c617 |
- if f is not None:
|
|
|
775c617 |
- f.close()
|
|
|
775c617 |
- except ImportError:
|
|
|
775c617 |
- return True
|
|
|
775c617 |
- return False
|
|
|
05485cd |
diff --git a/joblib/externals/loky/backend/reduction.py b/joblib/externals/loky/backend/reduction.py
|
|
|
05485cd |
index 5d5414a1a1..0bad5f637f 100644
|
|
|
05485cd |
--- a/joblib/externals/loky/backend/reduction.py
|
|
|
05485cd |
+++ b/joblib/externals/loky/backend/reduction.py
|
|
|
05485cd |
@@ -122,7 +122,7 @@ else:
|
|
|
775c617 |
|
|
|
775c617 |
# global variable to change the pickler behavior
|
|
|
775c617 |
try:
|
|
|
775c617 |
- from joblib.externals import cloudpickle # noqa: F401
|
|
|
775c617 |
+ import cloudpickle # noqa: F401
|
|
|
775c617 |
DEFAULT_ENV = "cloudpickle"
|
|
|
775c617 |
except ImportError:
|
|
|
775c617 |
# If cloudpickle is not present, fallback to pickle
|
|
|
05485cd |
@@ -149,7 +149,7 @@ def set_loky_pickler(loky_pickler=None):
|
|
|
775c617 |
return
|
|
|
775c617 |
|
|
|
775c617 |
if loky_pickler == "cloudpickle":
|
|
|
775c617 |
- from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls
|
|
|
775c617 |
+ from cloudpickle import CloudPickler as loky_pickler_cls
|
|
|
775c617 |
else:
|
|
|
775c617 |
try:
|
|
|
775c617 |
from importlib import import_module
|
|
|
05485cd |
diff --git a/joblib/externals/loky/cloudpickle_wrapper.py b/joblib/externals/loky/cloudpickle_wrapper.py
|
|
|
05485cd |
index 1bf41a336e..14603c5b76 100644
|
|
|
05485cd |
--- a/joblib/externals/loky/cloudpickle_wrapper.py
|
|
|
05485cd |
+++ b/joblib/externals/loky/cloudpickle_wrapper.py
|
|
|
05485cd |
@@ -2,7 +2,7 @@ import inspect
|
|
|
775c617 |
from functools import partial
|
|
|
775c617 |
|
|
|
775c617 |
try:
|
|
|
775c617 |
- from joblib.externals.cloudpickle import dumps, loads
|
|
|
775c617 |
+ from cloudpickle import dumps, loads
|
|
|
775c617 |
cloudpickle = True
|
|
|
775c617 |
except ImportError:
|
|
|
775c617 |
cloudpickle = False
|
|
|
05485cd |
diff --git a/joblib/parallel.py b/joblib/parallel.py
|
|
|
05485cd |
index 03dcd92a3f..124ea84598 100644
|
|
|
05485cd |
--- a/joblib/parallel.py
|
|
|
05485cd |
+++ b/joblib/parallel.py
|
|
|
05485cd |
@@ -29,7 +29,7 @@ from ._parallel_backends import (FallbackToBackend, MultiprocessingBackend,
|
|
|
775c617 |
ThreadingBackend, SequentialBackend,
|
|
|
775c617 |
LokyBackend)
|
|
|
775c617 |
from ._compat import _basestring
|
|
|
775c617 |
-from .externals.cloudpickle import dumps, loads
|
|
|
775c617 |
+from cloudpickle import dumps, loads
|
|
|
775c617 |
from .externals import loky
|
|
|
775c617 |
|
|
|
775c617 |
# Make sure that those two classes are part of the public joblib.parallel API
|
|
|
05485cd |
diff --git a/setup.py b/setup.py
|
|
|
05485cd |
index 7d60b210a3..5ede06c44d 100755
|
|
|
05485cd |
--- a/setup.py
|
|
|
05485cd |
+++ b/setup.py
|
|
|
05485cd |
@@ -53,6 +53,6 @@ if __name__ == '__main__':
|
|
|
775c617 |
'data/*.npy',
|
|
|
775c617 |
'data/*.npy.z']},
|
|
|
775c617 |
packages=['joblib', 'joblib.test', 'joblib.test.data',
|
|
|
775c617 |
- 'joblib.externals', 'joblib.externals.cloudpickle',
|
|
|
775c617 |
+ 'joblib.externals',
|
|
|
775c617 |
'joblib.externals.loky', 'joblib.externals.loky.backend'],
|
|
|
775c617 |
**extra_setuptools_args)
|