05485cd
From 59683c433fd4c19f4921e4b44b8a70174e82fc4b Mon Sep 17 00:00:00 2001
05485cd
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
05485cd
Date: Tue, 10 Sep 2019 08:32:24 +0200
05485cd
Subject: [PATCH 18/18] Unbundle cloudpickle
05485cd
05485cd
---
05485cd
 joblib/externals/cloudpickle/__init__.py     |   11 -
05485cd
 joblib/externals/cloudpickle/cloudpickle.py  | 1380 ------------------
05485cd
 joblib/externals/loky/backend/reduction.py   |    4 +-
05485cd
 joblib/externals/loky/cloudpickle_wrapper.py |    2 +-
05485cd
 joblib/parallel.py                           |    2 +-
05485cd
 setup.py                                     |    2 +-
05485cd
 6 files changed, 5 insertions(+), 1396 deletions(-)
05485cd
 delete mode 100644 joblib/externals/cloudpickle/__init__.py
05485cd
 delete mode 100644 joblib/externals/cloudpickle/cloudpickle.py
05485cd
05485cd
diff --git a/joblib/externals/cloudpickle/__init__.py b/joblib/externals/cloudpickle/__init__.py
05485cd
deleted file mode 100644
05485cd
index 9806b5c7b2..0000000000
05485cd
--- a/joblib/externals/cloudpickle/__init__.py
05485cd
+++ /dev/null
05485cd
@@ -1,11 +0,0 @@
05485cd
-from __future__ import absolute_import
05485cd
-
05485cd
-import sys
05485cd
-import pickle
05485cd
-
05485cd
-
05485cd
-from .cloudpickle import *
05485cd
-if sys.version_info[:2] >= (3, 8):
05485cd
-    from .cloudpickle_fast import CloudPickler, dumps, dump
05485cd
-
05485cd
-__version__ = '1.2.1'
05485cd
diff --git a/joblib/externals/cloudpickle/cloudpickle.py b/joblib/externals/cloudpickle/cloudpickle.py
05485cd
deleted file mode 100644
05485cd
index cc74c642ec..0000000000
05485cd
--- a/joblib/externals/cloudpickle/cloudpickle.py
05485cd
+++ /dev/null
05485cd
@@ -1,1380 +0,0 @@
775c617
-"""
775c617
-This class is defined to override standard pickle functionality
775c617
-
775c617
-The goals of it follow:
775c617
--Serialize lambdas and nested functions to compiled byte code
775c617
--Deal with main module correctly
775c617
--Deal with other non-serializable objects
775c617
-
775c617
-It does not include an unpickler, as standard python unpickling suffices.
775c617
-
775c617
-This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
775c617
-<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
775c617
-
775c617
-Copyright (c) 2012, Regents of the University of California.
775c617
-Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
775c617
-All rights reserved.
775c617
-
775c617
-Redistribution and use in source and binary forms, with or without
775c617
-modification, are permitted provided that the following conditions
775c617
-are met:
775c617
-    * Redistributions of source code must retain the above copyright
775c617
-      notice, this list of conditions and the following disclaimer.
775c617
-    * Redistributions in binary form must reproduce the above copyright
775c617
-      notice, this list of conditions and the following disclaimer in the
775c617
-      documentation and/or other materials provided with the distribution.
775c617
-    * Neither the name of the University of California, Berkeley nor the
775c617
-      names of its contributors may be used to endorse or promote
775c617
-      products derived from this software without specific prior written
775c617
-      permission.
775c617
-
775c617
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
775c617
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
775c617
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
775c617
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
775c617
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
775c617
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
775c617
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
775c617
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
775c617
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
775c617
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
775c617
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
775c617
-"""
775c617
-from __future__ import print_function
775c617
-
775c617
-import dis
775c617
-from functools import partial
775c617
-import io
775c617
-import itertools
775c617
-import logging
775c617
-import opcode
775c617
-import operator
775c617
-import pickle
05485cd
-import platform
775c617
-import struct
775c617
-import sys
775c617
-import traceback
775c617
-import types
775c617
-import weakref
05485cd
-import uuid
05485cd
-import threading
05485cd
-
05485cd
-
05485cd
-try:
05485cd
-    from enum import Enum
05485cd
-except ImportError:
05485cd
-    Enum = None
775c617
-
775c617
-# cloudpickle is meant for inter process communication: we expect all
775c617
-# communicating processes to run the same Python version hence we favor
775c617
-# communication speed over compatibility:
775c617
-DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
775c617
-
05485cd
-# Track the provenance of reconstructed dynamic classes to make it possible to
05485cd
-# recontruct instances from the matching singleton class definition when
05485cd
-# appropriate and preserve the usual "isinstance" semantics of Python objects.
05485cd
-_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
05485cd
-_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
05485cd
-_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
05485cd
-
05485cd
-PYPY = platform.python_implementation() == "PyPy"
05485cd
-
05485cd
-builtin_code_type = None
05485cd
-if PYPY:
05485cd
-    # builtin-code objects only exist in pypy
05485cd
-    builtin_code_type = type(float.__new__.__code__)
775c617
-
775c617
-if sys.version_info[0] < 3:  # pragma: no branch
775c617
-    from pickle import Pickler
775c617
-    try:
775c617
-        from cStringIO import StringIO
775c617
-    except ImportError:
775c617
-        from StringIO import StringIO
775c617
-    string_types = (basestring,)  # noqa
775c617
-    PY3 = False
05485cd
-    PY2 = True
775c617
-else:
775c617
-    types.ClassType = type
775c617
-    from pickle import _Pickler as Pickler
775c617
-    from io import BytesIO as StringIO
775c617
-    string_types = (str,)
775c617
-    PY3 = True
05485cd
-    PY2 = False
05485cd
-    from importlib._bootstrap import _find_spec
05485cd
-
05485cd
-_extract_code_globals_cache = weakref.WeakKeyDictionary()
05485cd
-
05485cd
-
05485cd
-def _ensure_tracking(class_def):
05485cd
-    with _DYNAMIC_CLASS_TRACKER_LOCK:
05485cd
-        class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
05485cd
-        if class_tracker_id is None:
05485cd
-            class_tracker_id = uuid.uuid4().hex
05485cd
-            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
05485cd
-            _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
05485cd
-    return class_tracker_id
05485cd
-
05485cd
-
05485cd
-def _lookup_class_or_track(class_tracker_id, class_def):
05485cd
-    if class_tracker_id is not None:
05485cd
-        with _DYNAMIC_CLASS_TRACKER_LOCK:
05485cd
-            class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
05485cd
-                class_tracker_id, class_def)
05485cd
-            _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
05485cd
-    return class_def
05485cd
-
05485cd
-if sys.version_info[:2] >= (3, 5):
05485cd
-    from pickle import _getattribute
05485cd
-elif sys.version_info[:2] >= (3, 4):
05485cd
-    from pickle import _getattribute as _py34_getattribute
05485cd
-    #  pickle._getattribute does not return the parent under Python 3.4
05485cd
-    def _getattribute(obj, name):
05485cd
-        return _py34_getattribute(obj, name), None
05485cd
-else:
05485cd
-    # pickle._getattribute is a python3 addition and enchancement of getattr,
05485cd
-    # that can handle dotted attribute names. In cloudpickle for python2,
05485cd
-    # handling dotted names is not needed, so we simply define _getattribute as
05485cd
-    # a wrapper around getattr.
05485cd
-    def _getattribute(obj, name):
05485cd
-        return getattr(obj, name, None), None
05485cd
-
05485cd
-
05485cd
-def _whichmodule(obj, name):
05485cd
-    """Find the module an object belongs to.
05485cd
-
05485cd
-    This function differs from ``pickle.whichmodule`` in two ways:
05485cd
-    - it does not mangle the cases where obj's module is __main__ and obj was
05485cd
-      not found in any module.
05485cd
-    - Errors arising during module introspection are ignored, as those errors
05485cd
-      are considered unwanted side effects.
05485cd
-    """
05485cd
-    module_name = getattr(obj, '__module__', None)
05485cd
-    if module_name is not None:
05485cd
-        return module_name
05485cd
-    # Protect the iteration by using a list copy of sys.modules against dynamic
05485cd
-    # modules that trigger imports of other modules upon calls to getattr.
05485cd
-    for module_name, module in list(sys.modules.items()):
05485cd
-        if module_name == '__main__' or module is None:
05485cd
-            continue
05485cd
-        try:
05485cd
-            if _getattribute(module, name)[0] is obj:
05485cd
-                return module_name
05485cd
-        except Exception:
05485cd
-            pass
05485cd
-    return None
05485cd
-
05485cd
-
05485cd
-def _is_global(obj, name=None):
05485cd
-    """Determine if obj can be pickled as attribute of a file-backed module"""
05485cd
-    if name is None:
05485cd
-        name = getattr(obj, '__qualname__', None)
05485cd
-    if name is None:
05485cd
-        name = getattr(obj, '__name__', None)
05485cd
-
05485cd
-    module_name = _whichmodule(obj, name)
05485cd
-
05485cd
-    if module_name is None:
05485cd
-        # In this case, obj.__module__ is None AND obj was not found in any
05485cd
-        # imported module. obj is thus treated as dynamic.
05485cd
-        return False
05485cd
-
05485cd
-    if module_name == "__main__":
05485cd
-        return False
05485cd
-
05485cd
-    module = sys.modules.get(module_name, None)
05485cd
-    if module is None:
05485cd
-        # The main reason why obj's module would not be imported is that this
05485cd
-        # module has been dynamically created, using for example
05485cd
-        # types.ModuleType. The other possibility is that module was removed
05485cd
-        # from sys.modules after obj was created/imported. But this case is not
05485cd
-        # supported, as the standard pickle does not support it either.
05485cd
-        return False
05485cd
-
05485cd
-    # module has been added to sys.modules, but it can still be dynamic.
05485cd
-    if _is_dynamic(module):
05485cd
-        return False
05485cd
-
05485cd
-    try:
05485cd
-        obj2, parent = _getattribute(module, name)
05485cd
-    except AttributeError:
05485cd
-        # obj was not found inside the module it points to
05485cd
-        return False
05485cd
-    return obj2 is obj
05485cd
-
05485cd
-
05485cd
-def _extract_code_globals(co):
05485cd
-    """
05485cd
-    Find all globals names read or written to by codeblock co
05485cd
-    """
05485cd
-    out_names = _extract_code_globals_cache.get(co)
05485cd
-    if out_names is None:
05485cd
-        names = co.co_names
05485cd
-        out_names = {names[oparg] for _, oparg in _walk_global_ops(co)}
05485cd
-
05485cd
-        # Declaring a function inside another one using the "def ..."
05485cd
-        # syntax generates a constant code object corresonding to the one
05485cd
-        # of the nested function's As the nested function may itself need
05485cd
-        # global variables, we need to introspect its code, extract its
05485cd
-        # globals, (look for code object in it's co_consts attribute..) and
05485cd
-        # add the result to code_globals
05485cd
-        if co.co_consts:
05485cd
-            for const in co.co_consts:
05485cd
-                if isinstance(const, types.CodeType):
05485cd
-                    out_names |= _extract_code_globals(const)
05485cd
-
05485cd
-        _extract_code_globals_cache[co] = out_names
05485cd
-
05485cd
-    return out_names
05485cd
-
05485cd
-
05485cd
-def _find_imported_submodules(code, top_level_dependencies):
05485cd
-    """
05485cd
-    Find currently imported submodules used by a function.
05485cd
-
05485cd
-    Submodules used by a function need to be detected and referenced for the
05485cd
-    function to work correctly at depickling time. Because submodules can be
05485cd
-    referenced as attribute of their parent package (``package.submodule``), we
05485cd
-    need a special introspection technique that does not rely on GLOBAL-related
05485cd
-    opcodes to find references of them in a code object.
05485cd
-
05485cd
-    Example:
05485cd
-    ```
05485cd
-    import concurrent.futures
05485cd
-    import cloudpickle
05485cd
-    def func():
05485cd
-        x = concurrent.futures.ThreadPoolExecutor
05485cd
-    if __name__ == '__main__':
05485cd
-        cloudpickle.dumps(func)
05485cd
-    ```
05485cd
-    The globals extracted by cloudpickle in the function's state include the
05485cd
-    concurrent package, but not its submodule (here, concurrent.futures), which
05485cd
-    is the module used by func. Find_imported_submodules will detect the usage
05485cd
-    of concurrent.futures. Saving this module alongside with func will ensure
05485cd
-    that calling func once depickled does not fail due to concurrent.futures
05485cd
-    not being imported
05485cd
-    """
05485cd
-
05485cd
-    subimports = []
05485cd
-    # check if any known dependency is an imported package
05485cd
-    for x in top_level_dependencies:
05485cd
-        if (isinstance(x, types.ModuleType) and
05485cd
-                hasattr(x, '__package__') and x.__package__):
05485cd
-            # check if the package has any currently loaded sub-imports
05485cd
-            prefix = x.__name__ + '.'
05485cd
-            # A concurrent thread could mutate sys.modules,
05485cd
-            # make sure we iterate over a copy to avoid exceptions
05485cd
-            for name in list(sys.modules):
05485cd
-                # Older versions of pytest will add a "None" module to
05485cd
-                # sys.modules.
05485cd
-                if name is not None and name.startswith(prefix):
05485cd
-                    # check whether the function can address the sub-module
05485cd
-                    tokens = set(name[len(prefix):].split('.'))
05485cd
-                    if not tokens - set(code.co_names):
05485cd
-                        subimports.append(sys.modules[name])
05485cd
-    return subimports
775c617
-
775c617
-
775c617
-def _make_cell_set_template_code():
775c617
-    """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
775c617
-
775c617
-    Notes
775c617
-    -----
775c617
-    In Python 3, we could use an easier function:
775c617
-
775c617
-    .. code-block:: python
775c617
-
775c617
-       def f():
775c617
-           cell = None
775c617
-
775c617
-           def _stub(value):
775c617
-               nonlocal cell
775c617
-               cell = value
775c617
-
775c617
-           return _stub
775c617
-
775c617
-        _cell_set_template_code = f().__code__
775c617
-
775c617
-    This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is
775c617
-    invalid syntax on Python 2. If we use this function we also don't need
775c617
-    to do the weird freevars/cellvars swap below
775c617
-    """
775c617
-    def inner(value):
775c617
-        lambda: cell  # make ``cell`` a closure so that we get a STORE_DEREF
775c617
-        cell = value
775c617
-
775c617
-    co = inner.__code__
775c617
-
775c617
-    # NOTE: we are marking the cell variable as a free variable intentionally
775c617
-    # so that we simulate an inner function instead of the outer function. This
775c617
-    # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
05485cd
-    if PY2:  # pragma: no branch
775c617
-        return types.CodeType(
775c617
-            co.co_argcount,
775c617
-            co.co_nlocals,
775c617
-            co.co_stacksize,
775c617
-            co.co_flags,
775c617
-            co.co_code,
775c617
-            co.co_consts,
775c617
-            co.co_names,
775c617
-            co.co_varnames,
775c617
-            co.co_filename,
775c617
-            co.co_name,
775c617
-            co.co_firstlineno,
775c617
-            co.co_lnotab,
775c617
-            co.co_cellvars,  # this is the trickery
775c617
-            (),
775c617
-        )
775c617
-    else:
05485cd
-        if hasattr(types.CodeType, "co_posonlyargcount"):  # pragma: no branch
05485cd
-            return types.CodeType(
05485cd
-                co.co_argcount,
05485cd
-                co.co_posonlyargcount,  # Python3.8 with PEP570
05485cd
-                co.co_kwonlyargcount,
05485cd
-                co.co_nlocals,
05485cd
-                co.co_stacksize,
05485cd
-                co.co_flags,
05485cd
-                co.co_code,
05485cd
-                co.co_consts,
05485cd
-                co.co_names,
05485cd
-                co.co_varnames,
05485cd
-                co.co_filename,
05485cd
-                co.co_name,
05485cd
-                co.co_firstlineno,
05485cd
-                co.co_lnotab,
05485cd
-                co.co_cellvars,  # this is the trickery
05485cd
-                (),
05485cd
-            )
05485cd
-        else:
05485cd
-            return types.CodeType(
05485cd
-                co.co_argcount,
05485cd
-                co.co_kwonlyargcount,
05485cd
-                co.co_nlocals,
05485cd
-                co.co_stacksize,
05485cd
-                co.co_flags,
05485cd
-                co.co_code,
05485cd
-                co.co_consts,
05485cd
-                co.co_names,
05485cd
-                co.co_varnames,
05485cd
-                co.co_filename,
05485cd
-                co.co_name,
05485cd
-                co.co_firstlineno,
05485cd
-                co.co_lnotab,
05485cd
-                co.co_cellvars,  # this is the trickery
05485cd
-                (),
05485cd
-            )
775c617
-
775c617
-_cell_set_template_code = _make_cell_set_template_code()
775c617
-
775c617
-
775c617
-def cell_set(cell, value):
775c617
-    """Set the value of a closure cell.
775c617
-    """
775c617
-    return types.FunctionType(
775c617
-        _cell_set_template_code,
775c617
-        {},
775c617
-        '_cell_set_inner',
775c617
-        (),
775c617
-        (cell,),
775c617
-    )(value)
775c617
-
775c617
-
775c617
-# relevant opcodes
775c617
-STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
775c617
-DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
775c617
-LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
775c617
-GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
775c617
-HAVE_ARGUMENT = dis.HAVE_ARGUMENT
775c617
-EXTENDED_ARG = dis.EXTENDED_ARG
775c617
-
775c617
-
775c617
-_BUILTIN_TYPE_NAMES = {}
775c617
-for k, v in types.__dict__.items():
775c617
-    if type(v) is type:
775c617
-        _BUILTIN_TYPE_NAMES[v] = k
775c617
-
775c617
-
775c617
-def _builtin_type(name):
775c617
-    return getattr(types, name)
775c617
-
775c617
-
775c617
-if sys.version_info < (3, 4):  # pragma: no branch
775c617
-    def _walk_global_ops(code):
775c617
-        """
775c617
-        Yield (opcode, argument number) tuples for all
775c617
-        global-referencing instructions in *code*.
775c617
-        """
775c617
-        code = getattr(code, 'co_code', b'')
05485cd
-        if PY2:  # pragma: no branch
775c617
-            code = map(ord, code)
775c617
-
775c617
-        n = len(code)
775c617
-        i = 0
775c617
-        extended_arg = 0
775c617
-        while i < n:
775c617
-            op = code[i]
775c617
-            i += 1
775c617
-            if op >= HAVE_ARGUMENT:
775c617
-                oparg = code[i] + code[i + 1] * 256 + extended_arg
775c617
-                extended_arg = 0
775c617
-                i += 2
775c617
-                if op == EXTENDED_ARG:
775c617
-                    extended_arg = oparg * 65536
775c617
-                if op in GLOBAL_OPS:
775c617
-                    yield op, oparg
775c617
-
775c617
-else:
775c617
-    def _walk_global_ops(code):
775c617
-        """
775c617
-        Yield (opcode, argument number) tuples for all
775c617
-        global-referencing instructions in *code*.
775c617
-        """
775c617
-        for instr in dis.get_instructions(code):
775c617
-            op = instr.opcode
775c617
-            if op in GLOBAL_OPS:
775c617
-                yield op, instr.arg
775c617
-
775c617
-
05485cd
-def _extract_class_dict(cls):
05485cd
-    """Retrieve a copy of the dict of a class without the inherited methods"""
05485cd
-    clsdict = dict(cls.__dict__)  # copy dict proxy to a dict
05485cd
-    if len(cls.__bases__) == 1:
05485cd
-        inherited_dict = cls.__bases__[0].__dict__
05485cd
-    else:
05485cd
-        inherited_dict = {}
05485cd
-        for base in reversed(cls.__bases__):
05485cd
-            inherited_dict.update(base.__dict__)
05485cd
-    to_remove = []
05485cd
-    for name, value in clsdict.items():
05485cd
-        try:
05485cd
-            base_value = inherited_dict[name]
05485cd
-            if value is base_value:
05485cd
-                to_remove.append(name)
05485cd
-        except KeyError:
05485cd
-            pass
05485cd
-    for name in to_remove:
05485cd
-        clsdict.pop(name)
05485cd
-    return clsdict
05485cd
-
05485cd
-
775c617
-class CloudPickler(Pickler):
775c617
-
775c617
-    dispatch = Pickler.dispatch.copy()
775c617
-
775c617
-    def __init__(self, file, protocol=None):
775c617
-        if protocol is None:
775c617
-            protocol = DEFAULT_PROTOCOL
775c617
-        Pickler.__init__(self, file, protocol=protocol)
775c617
-        # map ids to dictionary. used to ensure that functions can share global env
775c617
-        self.globals_ref = {}
775c617
-
775c617
-    def dump(self, obj):
775c617
-        self.inject_addons()
775c617
-        try:
775c617
-            return Pickler.dump(self, obj)
775c617
-        except RuntimeError as e:
775c617
-            if 'recursion' in e.args[0]:
775c617
-                msg = """Could not pickle object as excessively deep recursion required."""
775c617
-                raise pickle.PicklingError(msg)
775c617
-            else:
775c617
-                raise
775c617
-
775c617
-    def save_memoryview(self, obj):
775c617
-        self.save(obj.tobytes())
775c617
-
775c617
-    dispatch[memoryview] = save_memoryview
775c617
-
05485cd
-    if PY2:  # pragma: no branch
775c617
-        def save_buffer(self, obj):
775c617
-            self.save(str(obj))
775c617
-
775c617
-        dispatch[buffer] = save_buffer  # noqa: F821 'buffer' was removed in Python 3
775c617
-
775c617
-    def save_module(self, obj):
775c617
-        """
775c617
-        Save a module as an import
775c617
-        """
775c617
-        if _is_dynamic(obj):
775c617
-            self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
775c617
-                             obj=obj)
775c617
-        else:
775c617
-            self.save_reduce(subimport, (obj.__name__,), obj=obj)
775c617
-
775c617
-    dispatch[types.ModuleType] = save_module
775c617
-
775c617
-    def save_codeobject(self, obj):
775c617
-        """
775c617
-        Save a code object
775c617
-        """
775c617
-        if PY3:  # pragma: no branch
05485cd
-            if hasattr(obj, "co_posonlyargcount"):  # pragma: no branch
05485cd
-                args = (
05485cd
-                    obj.co_argcount, obj.co_posonlyargcount,
05485cd
-                    obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
05485cd
-                    obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
05485cd
-                    obj.co_varnames, obj.co_filename, obj.co_name,
05485cd
-                    obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
05485cd
-                    obj.co_cellvars
05485cd
-                )
05485cd
-            else:
05485cd
-                args = (
05485cd
-                    obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
05485cd
-                    obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
05485cd
-                    obj.co_names, obj.co_varnames, obj.co_filename,
05485cd
-                    obj.co_name, obj.co_firstlineno, obj.co_lnotab,
05485cd
-                    obj.co_freevars, obj.co_cellvars
05485cd
-                )
775c617
-        else:
775c617
-            args = (
775c617
-                obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,
775c617
-                obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name,
775c617
-                obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars
775c617
-            )
775c617
-        self.save_reduce(types.CodeType, args, obj=obj)
775c617
-
775c617
-    dispatch[types.CodeType] = save_codeobject
775c617
-
775c617
-    def save_function(self, obj, name=None):
775c617
-        """ Registered with the dispatch to handle all function types.
775c617
-
775c617
-        Determines what kind of function obj is (e.g. lambda, defined at
775c617
-        interactive prompt, etc) and handles the pickling appropriately.
775c617
-        """
05485cd
-        if _is_global(obj, name=name):
05485cd
-            return Pickler.save_global(self, obj, name=name)
05485cd
-        elif PYPY and isinstance(obj.__code__, builtin_code_type):
05485cd
-            return self.save_pypy_builtin_func(obj)
775c617
-        else:
05485cd
-            return self.save_function_tuple(obj)
775c617
-
775c617
-    dispatch[types.FunctionType] = save_function
775c617
-
05485cd
-    def save_pypy_builtin_func(self, obj):
05485cd
-        """Save pypy equivalent of builtin functions.
05485cd
-
05485cd
-        PyPy does not have the concept of builtin-functions. Instead,
05485cd
-        builtin-functions are simple function instances, but with a
05485cd
-        builtin-code attribute.
05485cd
-        Most of the time, builtin functions should be pickled by attribute. But
05485cd
-        PyPy has flaky support for __qualname__, so some builtin functions such
05485cd
-        as float.__new__ will be classified as dynamic. For this reason only,
05485cd
-        we created this special routine. Because builtin-functions are not
05485cd
-        expected to have closure or globals, there is no additional hack
05485cd
-        (compared the one already implemented in pickle) to protect ourselves
05485cd
-        from reference cycles. A simple (reconstructor, newargs, obj.__dict__)
05485cd
-        tuple is save_reduced.
05485cd
-
05485cd
-        Note also that PyPy improved their support for __qualname__ in v3.6, so
05485cd
-        this routing should be removed when cloudpickle supports only PyPy 3.6
05485cd
-        and later.
775c617
-        """
05485cd
-        rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
05485cd
-                                   obj.__defaults__, obj.__closure__),
05485cd
-              obj.__dict__)
05485cd
-        self.save_reduce(*rv, obj=obj)
775c617
-
05485cd
-    def _save_dynamic_enum(self, obj, clsdict):
05485cd
-        """Special handling for dynamic Enum subclasses
775c617
-
05485cd
-        Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the
05485cd
-        EnumMeta metaclass has complex initialization that makes the Enum
05485cd
-        subclasses hold references to their own instances.
05485cd
-        """
05485cd
-        members = dict((e.name, e.value) for e in obj)
775c617
-
05485cd
-        # Python 2.7 with enum34 can have no qualname:
05485cd
-        qualname = getattr(obj, "__qualname__", None)
775c617
-
05485cd
-        self.save_reduce(_make_skeleton_enum,
05485cd
-                         (obj.__bases__, obj.__name__, qualname, members,
05485cd
-                          obj.__module__, _ensure_tracking(obj), None),
05485cd
-                         obj=obj)
775c617
-
05485cd
-        # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
05485cd
-        # Those attributes are already handled by the metaclass.
05485cd
-        for attrname in ["_generate_next_value_", "_member_names_",
05485cd
-                         "_member_map_", "_member_type_",
05485cd
-                         "_value2member_map_"]:
05485cd
-            clsdict.pop(attrname, None)
05485cd
-        for member in members:
05485cd
-            clsdict.pop(member)
775c617
-
775c617
-    def save_dynamic_class(self, obj):
05485cd
-        """Save a class that can't be stored as module global.
775c617
-
775c617
-        This method is used to serialize classes that are defined inside
775c617
-        functions, or that otherwise can't be serialized as attribute lookups
775c617
-        from global modules.
775c617
-        """
05485cd
-        clsdict = _extract_class_dict(obj)
775c617
-        clsdict.pop('__weakref__', None)
775c617
-
775c617
-        # For ABCMeta in python3.7+, remove _abc_impl as it is not picklable.
775c617
-        # This is a fix which breaks the cache but this only makes the first
775c617
-        # calls to issubclass slower.
775c617
-        if "_abc_impl" in clsdict:
775c617
-            import abc
775c617
-            (registry, _, _, _) = abc._get_dump(obj)
775c617
-            clsdict["_abc_impl"] = [subclass_weakref()
775c617
-                                    for subclass_weakref in registry]
775c617
-
775c617
-        # On PyPy, __doc__ is a readonly attribute, so we need to include it in
775c617
-        # the initial skeleton class.  This is safe because we know that the
775c617
-        # doc can't participate in a cycle with the original class.
775c617
-        type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
775c617
-
775c617
-        if hasattr(obj, "__slots__"):
775c617
-            type_kwargs['__slots__'] = obj.__slots__
775c617
-            # pickle string length optimization: member descriptors of obj are
775c617
-            # created automatically from obj's __slots__ attribute, no need to
775c617
-            # save them in obj's state
775c617
-            if isinstance(obj.__slots__, string_types):
775c617
-                clsdict.pop(obj.__slots__)
775c617
-            else:
775c617
-                for k in obj.__slots__:
775c617
-                    clsdict.pop(k, None)
775c617
-
05485cd
-        # If type overrides __dict__ as a property, include it in the type
05485cd
-        # kwargs. In Python 2, we can't set this attribute after construction.
775c617
-        __dict__ = clsdict.pop('__dict__', None)
775c617
-        if isinstance(__dict__, property):
775c617
-            type_kwargs['__dict__'] = __dict__
775c617
-
775c617
-        save = self.save
775c617
-        write = self.write
775c617
-
775c617
-        # We write pickle instructions explicitly here to handle the
775c617
-        # possibility that the type object participates in a cycle with its own
775c617
-        # __dict__. We first write an empty "skeleton" version of the class and
775c617
-        # memoize it before writing the class' __dict__ itself. We then write
775c617
-        # instructions to "rehydrate" the skeleton class by restoring the
775c617
-        # attributes from the __dict__.
775c617
-        #
775c617
-        # A type can appear in a cycle with its __dict__ if an instance of the
775c617
-        # type appears in the type's __dict__ (which happens for the stdlib
775c617
-        # Enum class), or if the type defines methods that close over the name
775c617
-        # of the type, (which is common for Python 2-style super() calls).
775c617
-
775c617
-        # Push the rehydration function.
775c617
-        save(_rehydrate_skeleton_class)
775c617
-
775c617
-        # Mark the start of the args tuple for the rehydration function.
775c617
-        write(pickle.MARK)
775c617
-
775c617
-        # Create and memoize an skeleton class with obj's name and bases.
05485cd
-        if Enum is not None and issubclass(obj, Enum):
05485cd
-            # Special handling of Enum subclasses
05485cd
-            self._save_dynamic_enum(obj, clsdict)
05485cd
-        else:
05485cd
-            # "Regular" class definition:
05485cd
-            tp = type(obj)
05485cd
-            self.save_reduce(_make_skeleton_class,
05485cd
-                             (tp, obj.__name__, obj.__bases__, type_kwargs,
05485cd
-                              _ensure_tracking(obj), None),
05485cd
-                             obj=obj)
775c617
-
775c617
-        # Now save the rest of obj's __dict__. Any references to obj
775c617
-        # encountered while saving will point to the skeleton class.
775c617
-        save(clsdict)
775c617
-
775c617
-        # Write a tuple of (skeleton_class, clsdict).
775c617
-        write(pickle.TUPLE)
775c617
-
775c617
-        # Call _rehydrate_skeleton_class(skeleton_class, clsdict)
775c617
-        write(pickle.REDUCE)
775c617
-
775c617
-    def save_function_tuple(self, func):
775c617
-        """  Pickles an actual func object.
775c617
-
775c617
-        A func comprises: code, globals, defaults, closure, and dict.  We
775c617
-        extract and save these, injecting reducing functions at certain points
775c617
-        to recreate the func object.  Keep in mind that some of these pieces
775c617
-        can contain a ref to the func itself.  Thus, a naive save on these
775c617
-        pieces could trigger an infinite loop of save's.  To get around that,
775c617
-        we first create a skeleton func object using just the code (this is
775c617
-        safe, since this won't contain a ref to the func), and memoize it as
775c617
-        soon as it's created.  The other stuff can then be filled in later.
775c617
-        """
775c617
-        if is_tornado_coroutine(func):
775c617
-            self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,),
775c617
-                             obj=func)
775c617
-            return
775c617
-
775c617
-        save = self.save
775c617
-        write = self.write
775c617
-
775c617
-        code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)
775c617
-
775c617
-        save(_fill_function)  # skeleton function updater
775c617
-        write(pickle.MARK)    # beginning of tuple that _fill_function expects
775c617
-
05485cd
-        # Extract currently-imported submodules used by func. Storing these
05485cd
-        # modules in a smoke _cloudpickle_subimports attribute of the object's
05485cd
-        # state will trigger the side effect of importing these modules at
05485cd
-        # unpickling time (which is necessary for func to work correctly once
05485cd
-        # depickled)
05485cd
-        submodules = _find_imported_submodules(
775c617
-            code,
775c617
-            itertools.chain(f_globals.values(), closure_values or ()),
775c617
-        )
775c617
-
775c617
-        # create a skeleton function object and memoize it
775c617
-        save(_make_skel_func)
775c617
-        save((
775c617
-            code,
775c617
-            len(closure_values) if closure_values is not None else -1,
775c617
-            base_globals,
775c617
-        ))
775c617
-        write(pickle.REDUCE)
775c617
-        self.memoize(func)
775c617
-
775c617
-        # save the rest of the func data needed by _fill_function
775c617
-        state = {
775c617
-            'globals': f_globals,
775c617
-            'defaults': defaults,
775c617
-            'dict': dct,
775c617
-            'closure_values': closure_values,
775c617
-            'module': func.__module__,
775c617
-            'name': func.__name__,
775c617
-            'doc': func.__doc__,
05485cd
-            '_cloudpickle_submodules': submodules
775c617
-        }
05485cd
-        if hasattr(func, '__annotations__') and sys.version_info >= (3, 4):
775c617
-            state['annotations'] = func.__annotations__
775c617
-        if hasattr(func, '__qualname__'):
775c617
-            state['qualname'] = func.__qualname__
05485cd
-        if hasattr(func, '__kwdefaults__'):
05485cd
-            state['kwdefaults'] = func.__kwdefaults__
775c617
-        save(state)
775c617
-        write(pickle.TUPLE)
775c617
-        write(pickle.REDUCE)  # applies _fill_function on the tuple
775c617
-
775c617
-    def extract_func_data(self, func):
775c617
-        """
775c617
-        Turn the function into a tuple of data necessary to recreate it:
775c617
-            code, globals, defaults, closure_values, dict
775c617
-        """
775c617
-        code = func.__code__
775c617
-
775c617
-        # extract all global ref's
05485cd
-        func_global_refs = _extract_code_globals(code)
775c617
-
775c617
-        # process all variables referenced by global environment
775c617
-        f_globals = {}
775c617
-        for var in func_global_refs:
775c617
-            if var in func.__globals__:
775c617
-                f_globals[var] = func.__globals__[var]
775c617
-
775c617
-        # defaults requires no processing
775c617
-        defaults = func.__defaults__
775c617
-
775c617
-        # process closure
775c617
-        closure = (
775c617
-            list(map(_get_cell_contents, func.__closure__))
775c617
-            if func.__closure__ is not None
775c617
-            else None
775c617
-        )
775c617
-
775c617
-        # save the dict
775c617
-        dct = func.__dict__
775c617
-
775c617
-        # base_globals represents the future global namespace of func at
775c617
-        # unpickling time. Looking it up and storing it in globals_ref allow
775c617
-        # functions sharing the same globals at pickling time to also
775c617
-        # share them once unpickled, at one condition: since globals_ref is
775c617
-        # an attribute of a Cloudpickler instance, and that a new CloudPickler is
775c617
-        # created each time pickle.dump or pickle.dumps is called, functions
775c617
-        # also need to be saved within the same invokation of
775c617
-        # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There
775c617
-        # is no such limitation when using Cloudpickler.dump, as long as the
775c617
-        # multiple invokations are bound to the same Cloudpickler.
775c617
-        base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
775c617
-
05485cd
-        if base_globals == {}:
05485cd
-            # Add module attributes used to resolve relative imports
05485cd
-            # instructions inside func.
05485cd
-            for k in ["__package__", "__name__", "__path__", "__file__"]:
05485cd
-                # Some built-in functions/methods such as object.__new__  have
05485cd
-                # their __globals__ set to None in PyPy
05485cd
-                if func.__globals__ is not None and k in func.__globals__:
05485cd
-                    base_globals[k] = func.__globals__[k]
775c617
-
05485cd
-        return (code, f_globals, defaults, closure, dct, base_globals)
775c617
-
05485cd
-    if not PY3:  # pragma: no branch
05485cd
-        # Python3 comes with native reducers that allow builtin functions and
05485cd
-        # methods pickling as module/class attributes.  The following method
05485cd
-        # extends this for python2.
05485cd
-        # Please note that currently, neither pickle nor cloudpickle support
05485cd
-        # dynamically created builtin functions/method pickling.
05485cd
-        def save_builtin_function_or_method(self, obj):
05485cd
-            is_bound = getattr(obj, '__self__', None) is not None
05485cd
-            if is_bound:
05485cd
-                # obj is a bound builtin method.
05485cd
-                rv = (getattr, (obj.__self__, obj.__name__))
05485cd
-                return self.save_reduce(obj=obj, *rv)
05485cd
-
05485cd
-            is_unbound = hasattr(obj, '__objclass__')
05485cd
-            if is_unbound:
05485cd
-                # obj is an unbound builtin method (accessed from its class)
05485cd
-                rv = (getattr, (obj.__objclass__, obj.__name__))
05485cd
-                return self.save_reduce(obj=obj, *rv)
05485cd
-
05485cd
-            # Otherwise, obj is not a method, but a function. Fallback to
05485cd
-            # default pickling by attribute.
05485cd
-            return Pickler.save_global(self, obj)
05485cd
-
05485cd
-        dispatch[types.BuiltinFunctionType] = save_builtin_function_or_method
05485cd
-
05485cd
-        # A comprehensive summary of the various kinds of builtin methods can
05485cd
-        # be found in PEP 579: https://www.python.org/dev/peps/pep-0579/
05485cd
-        classmethod_descriptor_type = type(float.__dict__['fromhex'])
05485cd
-        wrapper_descriptor_type = type(float.__repr__)
05485cd
-        method_wrapper_type = type(1.5.__repr__)
05485cd
-
05485cd
-        dispatch[classmethod_descriptor_type] = save_builtin_function_or_method
05485cd
-        dispatch[wrapper_descriptor_type] = save_builtin_function_or_method
05485cd
-        dispatch[method_wrapper_type] = save_builtin_function_or_method
05485cd
-
05485cd
-    if sys.version_info[:2] < (3, 4):
05485cd
-        method_descriptor = type(str.upper)
05485cd
-        dispatch[method_descriptor] = save_builtin_function_or_method
775c617
-
775c617
-    def save_global(self, obj, name=None, pack=struct.pack):
775c617
-        """
775c617
-        Save a "global".
775c617
-
775c617
-        The name of this method is somewhat misleading: all types get
775c617
-        dispatched here.
775c617
-        """
775c617
-        if obj is type(None):
775c617
-            return self.save_reduce(type, (None,), obj=obj)
775c617
-        elif obj is type(Ellipsis):
775c617
-            return self.save_reduce(type, (Ellipsis,), obj=obj)
775c617
-        elif obj is type(NotImplemented):
775c617
-            return self.save_reduce(type, (NotImplemented,), obj=obj)
05485cd
-        elif obj in _BUILTIN_TYPE_NAMES:
05485cd
-            return self.save_reduce(
05485cd
-                _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
05485cd
-        elif name is not None:
05485cd
-            Pickler.save_global(self, obj, name=name)
05485cd
-        elif not _is_global(obj, name=name):
05485cd
-            self.save_dynamic_class(obj)
05485cd
-        else:
05485cd
-            Pickler.save_global(self, obj, name=name)
775c617
-
775c617
-    dispatch[type] = save_global
775c617
-    dispatch[types.ClassType] = save_global
775c617
-
775c617
-    def save_instancemethod(self, obj):
775c617
-        # Memoization rarely is ever useful due to python bounding
775c617
-        if obj.__self__ is None:
775c617
-            self.save_reduce(getattr, (obj.im_class, obj.__name__))
775c617
-        else:
775c617
-            if PY3:  # pragma: no branch
775c617
-                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)
775c617
-            else:
775c617
-                self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__),
775c617
-                                 obj=obj)
775c617
-
775c617
-    dispatch[types.MethodType] = save_instancemethod
775c617
-
775c617
-    def save_inst(self, obj):
775c617
-        """Inner logic to save instance. Based off pickle.save_inst"""
775c617
-        cls = obj.__class__
775c617
-
775c617
-        # Try the dispatch table (pickle module doesn't do it)
775c617
-        f = self.dispatch.get(cls)
775c617
-        if f:
775c617
-            f(self, obj)  # Call unbound method with explicit self
775c617
-            return
775c617
-
775c617
-        memo = self.memo
775c617
-        write = self.write
775c617
-        save = self.save
775c617
-
775c617
-        if hasattr(obj, '__getinitargs__'):
775c617
-            args = obj.__getinitargs__()
775c617
-            len(args)  # XXX Assert it's a sequence
775c617
-            pickle._keep_alive(args, memo)
775c617
-        else:
775c617
-            args = ()
775c617
-
775c617
-        write(pickle.MARK)
775c617
-
775c617
-        if self.bin:
775c617
-            save(cls)
775c617
-            for arg in args:
775c617
-                save(arg)
775c617
-            write(pickle.OBJ)
775c617
-        else:
775c617
-            for arg in args:
775c617
-                save(arg)
775c617
-            write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n')
775c617
-
775c617
-        self.memoize(obj)
775c617
-
775c617
-        try:
775c617
-            getstate = obj.__getstate__
775c617
-        except AttributeError:
775c617
-            stuff = obj.__dict__
775c617
-        else:
775c617
-            stuff = getstate()
775c617
-            pickle._keep_alive(stuff, memo)
775c617
-        save(stuff)
775c617
-        write(pickle.BUILD)
775c617
-
05485cd
-    if PY2:  # pragma: no branch
775c617
-        dispatch[types.InstanceType] = save_inst
775c617
-
775c617
-    def save_property(self, obj):
775c617
-        # properties not correctly saved in python
775c617
-        self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj)
775c617
-
775c617
-    dispatch[property] = save_property
775c617
-
775c617
-    def save_classmethod(self, obj):
775c617
-        orig_func = obj.__func__
775c617
-        self.save_reduce(type(obj), (orig_func,), obj=obj)
775c617
-
775c617
-    dispatch[classmethod] = save_classmethod
775c617
-    dispatch[staticmethod] = save_classmethod
775c617
-
775c617
-    def save_itemgetter(self, obj):
775c617
-        """itemgetter serializer (needed for namedtuple support)"""
775c617
-        class Dummy:
775c617
-            def __getitem__(self, item):
775c617
-                return item
775c617
-        items = obj(Dummy())
775c617
-        if not isinstance(items, tuple):
775c617
-            items = (items,)
775c617
-        return self.save_reduce(operator.itemgetter, items)
775c617
-
775c617
-    if type(operator.itemgetter) is type:
775c617
-        dispatch[operator.itemgetter] = save_itemgetter
775c617
-
775c617
-    def save_attrgetter(self, obj):
775c617
-        """attrgetter serializer"""
775c617
-        class Dummy(object):
775c617
-            def __init__(self, attrs, index=None):
775c617
-                self.attrs = attrs
775c617
-                self.index = index
775c617
-            def __getattribute__(self, item):
775c617
-                attrs = object.__getattribute__(self, "attrs")
775c617
-                index = object.__getattribute__(self, "index")
775c617
-                if index is None:
775c617
-                    index = len(attrs)
775c617
-                    attrs.append(item)
775c617
-                else:
775c617
-                    attrs[index] = ".".join([attrs[index], item])
775c617
-                return type(self)(attrs, index)
775c617
-        attrs = []
775c617
-        obj(Dummy(attrs))
775c617
-        return self.save_reduce(operator.attrgetter, tuple(attrs))
775c617
-
775c617
-    if type(operator.attrgetter) is type:
775c617
-        dispatch[operator.attrgetter] = save_attrgetter
775c617
-
775c617
-    def save_file(self, obj):
775c617
-        """Save a file"""
775c617
-        try:
775c617
-            import StringIO as pystringIO  # we can't use cStringIO as it lacks the name attribute
775c617
-        except ImportError:
775c617
-            import io as pystringIO
775c617
-
775c617
-        if not hasattr(obj, 'name') or not hasattr(obj, 'mode'):
775c617
-            raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
775c617
-        if obj is sys.stdout:
775c617
-            return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
775c617
-        if obj is sys.stderr:
775c617
-            return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
775c617
-        if obj is sys.stdin:
775c617
-            raise pickle.PicklingError("Cannot pickle standard input")
775c617
-        if obj.closed:
775c617
-            raise pickle.PicklingError("Cannot pickle closed files")
775c617
-        if hasattr(obj, 'isatty') and obj.isatty():
775c617
-            raise pickle.PicklingError("Cannot pickle files that map to tty objects")
775c617
-        if 'r' not in obj.mode and '+' not in obj.mode:
775c617
-            raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)
775c617
-
775c617
-        name = obj.name
775c617
-
775c617
-        retval = pystringIO.StringIO()
775c617
-
775c617
-        try:
775c617
-            # Read the whole file
775c617
-            curloc = obj.tell()
775c617
-            obj.seek(0)
775c617
-            contents = obj.read()
775c617
-            obj.seek(curloc)
775c617
-        except IOError:
775c617
-            raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
775c617
-        retval.write(contents)
775c617
-        retval.seek(curloc)
775c617
-
775c617
-        retval.name = name
775c617
-        self.save(retval)
775c617
-        self.memoize(obj)
775c617
-
775c617
-    def save_ellipsis(self, obj):
775c617
-        self.save_reduce(_gen_ellipsis, ())
775c617
-
775c617
-    def save_not_implemented(self, obj):
775c617
-        self.save_reduce(_gen_not_implemented, ())
775c617
-
775c617
-    try:               # Python 2
775c617
-        dispatch[file] = save_file
775c617
-    except NameError:  # Python 3  # pragma: no branch
775c617
-        dispatch[io.TextIOWrapper] = save_file
775c617
-
775c617
-    dispatch[type(Ellipsis)] = save_ellipsis
775c617
-    dispatch[type(NotImplemented)] = save_not_implemented
775c617
-
775c617
-    def save_weakset(self, obj):
775c617
-        self.save_reduce(weakref.WeakSet, (list(obj),))
775c617
-
775c617
-    dispatch[weakref.WeakSet] = save_weakset
775c617
-
775c617
-    def save_logger(self, obj):
775c617
-        self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
775c617
-
775c617
-    dispatch[logging.Logger] = save_logger
775c617
-
775c617
-    def save_root_logger(self, obj):
775c617
-        self.save_reduce(logging.getLogger, (), obj=obj)
775c617
-
775c617
-    dispatch[logging.RootLogger] = save_root_logger
775c617
-
775c617
-    if hasattr(types, "MappingProxyType"):  # pragma: no branch
775c617
-        def save_mappingproxy(self, obj):
775c617
-            self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
775c617
-
775c617
-        dispatch[types.MappingProxyType] = save_mappingproxy
775c617
-
775c617
-    """Special functions for Add-on libraries"""
775c617
-    def inject_addons(self):
775c617
-        """Plug in system. Register additional pickling functions if modules already loaded"""
775c617
-        pass
775c617
-
775c617
-
775c617
-# Tornado support
775c617
-
775c617
-def is_tornado_coroutine(func):
775c617
-    """
775c617
-    Return whether *func* is a Tornado coroutine function.
775c617
-    Running coroutines are not supported.
775c617
-    """
775c617
-    if 'tornado.gen' not in sys.modules:
775c617
-        return False
775c617
-    gen = sys.modules['tornado.gen']
775c617
-    if not hasattr(gen, "is_coroutine_function"):
775c617
-        # Tornado version is too old
775c617
-        return False
775c617
-    return gen.is_coroutine_function(func)
775c617
-
775c617
-
775c617
-def _rebuild_tornado_coroutine(func):
775c617
-    from tornado import gen
775c617
-    return gen.coroutine(func)
775c617
-
775c617
-
775c617
-# Shorthands for legacy support
775c617
-
775c617
-def dump(obj, file, protocol=None):
775c617
-    """Serialize obj as bytes streamed into file
775c617
-
775c617
-    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
775c617
-    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
775c617
-    between processes running the same Python version.
775c617
-
775c617
-    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
775c617
-    compatibility with older versions of Python.
775c617
-    """
775c617
-    CloudPickler(file, protocol=protocol).dump(obj)
775c617
-
775c617
-
775c617
-def dumps(obj, protocol=None):
775c617
-    """Serialize obj as a string of bytes allocated in memory
775c617
-
775c617
-    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
775c617
-    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
775c617
-    between processes running the same Python version.
775c617
-
775c617
-    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
775c617
-    compatibility with older versions of Python.
775c617
-    """
775c617
-    file = StringIO()
775c617
-    try:
775c617
-        cp = CloudPickler(file, protocol=protocol)
775c617
-        cp.dump(obj)
775c617
-        return file.getvalue()
775c617
-    finally:
775c617
-        file.close()
775c617
-
775c617
-
775c617
-# including pickles unloading functions in this namespace
775c617
-load = pickle.load
775c617
-loads = pickle.loads
775c617
-
775c617
-
775c617
-# hack for __import__ not working as desired
775c617
-def subimport(name):
775c617
-    __import__(name)
775c617
-    return sys.modules[name]
775c617
-
775c617
-
775c617
-def dynamic_subimport(name, vars):
775c617
-    mod = types.ModuleType(name)
775c617
-    mod.__dict__.update(vars)
775c617
-    return mod
775c617
-
775c617
-
775c617
-def _gen_ellipsis():
775c617
-    return Ellipsis
775c617
-
775c617
-
775c617
-def _gen_not_implemented():
775c617
-    return NotImplemented
775c617
-
775c617
-
775c617
-def _get_cell_contents(cell):
775c617
-    try:
775c617
-        return cell.cell_contents
775c617
-    except ValueError:
775c617
-        # sentinel used by ``_fill_function`` which will leave the cell empty
775c617
-        return _empty_cell_value
775c617
-
775c617
-
775c617
-def instance(cls):
775c617
-    """Create a new instance of a class.
775c617
-
775c617
-    Parameters
775c617
-    ----------
775c617
-    cls : type
775c617
-        The class to create an instance of.
775c617
-
775c617
-    Returns
775c617
-    -------
775c617
-    instance : cls
775c617
-        A new instance of ``cls``.
775c617
-    """
775c617
-    return cls()
775c617
-
775c617
-
775c617
-@instance
775c617
-class _empty_cell_value(object):
775c617
-    """sentinel for empty closures
775c617
-    """
775c617
-    @classmethod
775c617
-    def __reduce__(cls):
775c617
-        return cls.__name__
775c617
-
775c617
-
775c617
-def _fill_function(*args):
775c617
-    """Fills in the rest of function data into the skeleton function object
775c617
-
775c617
-    The skeleton itself is create by _make_skel_func().
775c617
-    """
775c617
-    if len(args) == 2:
775c617
-        func = args[0]
775c617
-        state = args[1]
775c617
-    elif len(args) == 5:
775c617
-        # Backwards compat for cloudpickle v0.4.0, after which the `module`
775c617
-        # argument was introduced
775c617
-        func = args[0]
775c617
-        keys = ['globals', 'defaults', 'dict', 'closure_values']
775c617
-        state = dict(zip(keys, args[1:]))
775c617
-    elif len(args) == 6:
775c617
-        # Backwards compat for cloudpickle v0.4.1, after which the function
775c617
-        # state was passed as a dict to the _fill_function it-self.
775c617
-        func = args[0]
775c617
-        keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
775c617
-        state = dict(zip(keys, args[1:]))
775c617
-    else:
775c617
-        raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
775c617
-
775c617
-    # - At pickling time, any dynamic global variable used by func is
775c617
-    #   serialized by value (in state['globals']).
775c617
-    # - At unpickling time, func's __globals__ attribute is initialized by
775c617
-    #   first retrieving an empty isolated namespace that will be shared
775c617
-    #   with other functions pickled from the same original module
775c617
-    #   by the same CloudPickler instance and then updated with the
775c617
-    #   content of state['globals'] to populate the shared isolated
775c617
-    #   namespace with all the global variables that are specifically
775c617
-    #   referenced for this function.
775c617
-    func.__globals__.update(state['globals'])
775c617
-
775c617
-    func.__defaults__ = state['defaults']
775c617
-    func.__dict__ = state['dict']
775c617
-    if 'annotations' in state:
775c617
-        func.__annotations__ = state['annotations']
775c617
-    if 'doc' in state:
775c617
-        func.__doc__  = state['doc']
775c617
-    if 'name' in state:
775c617
-        func.__name__ = state['name']
775c617
-    if 'module' in state:
775c617
-        func.__module__ = state['module']
775c617
-    if 'qualname' in state:
775c617
-        func.__qualname__ = state['qualname']
05485cd
-    if 'kwdefaults' in state:
05485cd
-        func.__kwdefaults__ = state['kwdefaults']
05485cd
-    # _cloudpickle_subimports is a set of submodules that must be loaded for
05485cd
-    # the pickled function to work correctly at unpickling time. Now that these
05485cd
-    # submodules are depickled (hence imported), they can be removed from the
05485cd
-    # object's state (the object state only served as a reference holder to
05485cd
-    # these submodules)
05485cd
-    if '_cloudpickle_submodules' in state:
05485cd
-        state.pop('_cloudpickle_submodules')
775c617
-
775c617
-    cells = func.__closure__
775c617
-    if cells is not None:
775c617
-        for cell, value in zip(cells, state['closure_values']):
775c617
-            if value is not _empty_cell_value:
775c617
-                cell_set(cell, value)
775c617
-
775c617
-    return func
775c617
-
775c617
-
775c617
-def _make_empty_cell():
775c617
-    if False:
775c617
-        # trick the compiler into creating an empty cell in our lambda
775c617
-        cell = None
775c617
-        raise AssertionError('this route should not be executed')
775c617
-
775c617
-    return (lambda: cell).__closure__[0]
775c617
-
775c617
-
775c617
-def _make_skel_func(code, cell_count, base_globals=None):
775c617
-    """ Creates a skeleton function object that contains just the provided
775c617
-        code and the correct number of cells in func_closure.  All other
775c617
-        func attributes (e.g. func_globals) are empty.
775c617
-    """
775c617
-    # This is backward-compatibility code: for cloudpickle versions between
775c617
-    # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
775c617
-    # should now always be a dictionary.
775c617
-    if base_globals is None or isinstance(base_globals, str):
775c617
-        base_globals = {}
775c617
-
775c617
-    base_globals['__builtins__'] = __builtins__
775c617
-
775c617
-    closure = (
775c617
-        tuple(_make_empty_cell() for _ in range(cell_count))
775c617
-        if cell_count >= 0 else
775c617
-        None
775c617
-    )
775c617
-    return types.FunctionType(code, base_globals, None, None, closure)
775c617
-
775c617
-
05485cd
-def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
05485cd
-                         class_tracker_id, extra):
05485cd
-    """Build dynamic class with an empty __dict__ to be filled once memoized
05485cd
-
05485cd
-    If class_tracker_id is not None, try to lookup an existing class definition
05485cd
-    matching that id. If none is found, track a newly reconstructed class
05485cd
-    definition under that id so that other instances stemming from the same
05485cd
-    class id will also reuse this class definition.
05485cd
-
05485cd
-    The "extra" variable is meant to be a dict (or None) that can be used for
05485cd
-    forward compatibility shall the need arise.
05485cd
-    """
05485cd
-    skeleton_class = type_constructor(name, bases, type_kwargs)
05485cd
-    return _lookup_class_or_track(class_tracker_id, skeleton_class)
05485cd
-
05485cd
-
775c617
-def _rehydrate_skeleton_class(skeleton_class, class_dict):
775c617
-    """Put attributes from `class_dict` back on `skeleton_class`.
775c617
-
775c617
-    See CloudPickler.save_dynamic_class for more info.
775c617
-    """
775c617
-    registry = None
775c617
-    for attrname, attr in class_dict.items():
775c617
-        if attrname == "_abc_impl":
775c617
-            registry = attr
775c617
-        else:
775c617
-            setattr(skeleton_class, attrname, attr)
775c617
-    if registry is not None:
775c617
-        for subclass in registry:
775c617
-            skeleton_class.register(subclass)
775c617
-
775c617
-    return skeleton_class
775c617
-
775c617
-
05485cd
-def _make_skeleton_enum(bases, name, qualname, members, module,
05485cd
-                        class_tracker_id, extra):
05485cd
-    """Build dynamic enum with an empty __dict__ to be filled once memoized
05485cd
-
05485cd
-    The creation of the enum class is inspired by the code of
05485cd
-    EnumMeta._create_.
05485cd
-
05485cd
-    If class_tracker_id is not None, try to lookup an existing enum definition
05485cd
-    matching that id. If none is found, track a newly reconstructed enum
05485cd
-    definition under that id so that other instances stemming from the same
05485cd
-    class id will also reuse this enum definition.
05485cd
-
05485cd
-    The "extra" variable is meant to be a dict (or None) that can be used for
05485cd
-    forward compatibility shall the need arise.
05485cd
-    """
05485cd
-    # enums always inherit from their base Enum class at the last position in
05485cd
-    # the list of base classes:
05485cd
-    enum_base = bases[-1]
05485cd
-    metacls = enum_base.__class__
05485cd
-    classdict = metacls.__prepare__(name, bases)
05485cd
-
05485cd
-    for member_name, member_value in members.items():
05485cd
-        classdict[member_name] = member_value
05485cd
-    enum_class = metacls.__new__(metacls, name, bases, classdict)
05485cd
-    enum_class.__module__ = module
05485cd
-
05485cd
-    # Python 2.7 compat
05485cd
-    if qualname is not None:
05485cd
-        enum_class.__qualname__ = qualname
05485cd
-
05485cd
-    return _lookup_class_or_track(class_tracker_id, enum_class)
05485cd
-
05485cd
-
775c617
-def _is_dynamic(module):
775c617
-    """
775c617
-    Return True if the module is special module that cannot be imported by its
775c617
-    name.
775c617
-    """
775c617
-    # Quick check: module that have __file__ attribute are not dynamic modules.
775c617
-    if hasattr(module, '__file__'):
775c617
-        return False
775c617
-
775c617
-    if hasattr(module, '__spec__'):
05485cd
-        if module.__spec__ is not None:
05485cd
-            return False
05485cd
-
05485cd
-        # In PyPy, Some built-in modules such as _codecs can have their
05485cd
-        # __spec__ attribute set to None despite being imported.  For such
05485cd
-        # modules, the ``_find_spec`` utility of the standard library is used.
05485cd
-        parent_name = module.__name__.rpartition('.')[0]
05485cd
-        if parent_name:  # pragma: no cover
05485cd
-            # This code handles the case where an imported package (and not
05485cd
-            # module) remains with __spec__ set to None. It is however untested
05485cd
-            # as no package in the PyPy stdlib has __spec__ set to None after
05485cd
-            # it is imported.
05485cd
-            try:
05485cd
-                parent = sys.modules[parent_name]
05485cd
-            except KeyError:
05485cd
-                msg = "parent {!r} not in sys.modules"
05485cd
-                raise ImportError(msg.format(parent_name))
05485cd
-            else:
05485cd
-                pkgpath = parent.__path__
05485cd
-        else:
05485cd
-            pkgpath = None
05485cd
-        return _find_spec(module.__name__, pkgpath, module) is None
05485cd
-
775c617
-    else:
775c617
-        # Backward compat for Python 2
775c617
-        import imp
775c617
-        try:
775c617
-            path = None
775c617
-            for part in module.__name__.split('.'):
775c617
-                if path is not None:
775c617
-                    path = [path]
775c617
-                f, path, description = imp.find_module(part, path)
775c617
-                if f is not None:
775c617
-                    f.close()
775c617
-        except ImportError:
775c617
-            return True
775c617
-        return False
05485cd
diff --git a/joblib/externals/loky/backend/reduction.py b/joblib/externals/loky/backend/reduction.py
05485cd
index 5d5414a1a1..0bad5f637f 100644
05485cd
--- a/joblib/externals/loky/backend/reduction.py
05485cd
+++ b/joblib/externals/loky/backend/reduction.py
05485cd
@@ -122,7 +122,7 @@ else:
775c617
 
775c617
 # global variable to change the pickler behavior
775c617
 try:
775c617
-    from joblib.externals import cloudpickle  # noqa: F401
775c617
+    import cloudpickle  # noqa: F401
775c617
     DEFAULT_ENV = "cloudpickle"
775c617
 except ImportError:
775c617
     # If cloudpickle is not present, fallback to pickle
05485cd
@@ -149,7 +149,7 @@ def set_loky_pickler(loky_pickler=None):
775c617
         return
775c617
 
775c617
     if loky_pickler == "cloudpickle":
775c617
-        from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls
775c617
+        from cloudpickle import CloudPickler as loky_pickler_cls
775c617
     else:
775c617
         try:
775c617
             from importlib import import_module
05485cd
diff --git a/joblib/externals/loky/cloudpickle_wrapper.py b/joblib/externals/loky/cloudpickle_wrapper.py
05485cd
index 1bf41a336e..14603c5b76 100644
05485cd
--- a/joblib/externals/loky/cloudpickle_wrapper.py
05485cd
+++ b/joblib/externals/loky/cloudpickle_wrapper.py
05485cd
@@ -2,7 +2,7 @@ import inspect
775c617
 from functools import partial
775c617
 
775c617
 try:
775c617
-    from joblib.externals.cloudpickle import dumps, loads
775c617
+    from cloudpickle import dumps, loads
775c617
     cloudpickle = True
775c617
 except ImportError:
775c617
     cloudpickle = False
05485cd
diff --git a/joblib/parallel.py b/joblib/parallel.py
05485cd
index 03dcd92a3f..124ea84598 100644
05485cd
--- a/joblib/parallel.py
05485cd
+++ b/joblib/parallel.py
05485cd
@@ -29,7 +29,7 @@ from ._parallel_backends import (FallbackToBackend, MultiprocessingBackend,
775c617
                                  ThreadingBackend, SequentialBackend,
775c617
                                  LokyBackend)
775c617
 from ._compat import _basestring
775c617
-from .externals.cloudpickle import dumps, loads
775c617
+from cloudpickle import dumps, loads
775c617
 from .externals import loky
775c617
 
775c617
 # Make sure that those two classes are part of the public joblib.parallel API
05485cd
diff --git a/setup.py b/setup.py
05485cd
index 7d60b210a3..5ede06c44d 100755
05485cd
--- a/setup.py
05485cd
+++ b/setup.py
05485cd
@@ -53,6 +53,6 @@ if __name__ == '__main__':
775c617
                                         'data/*.npy',
775c617
                                         'data/*.npy.z']},
775c617
           packages=['joblib', 'joblib.test', 'joblib.test.data',
775c617
-                    'joblib.externals', 'joblib.externals.cloudpickle',
775c617
+                    'joblib.externals',
775c617
                     'joblib.externals.loky', 'joblib.externals.loky.backend'],
775c617
           **extra_setuptools_args)