Comments (10)
Here's a simple case:
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
# obj.x, obj.y, obj.z = x,y,z
return obj
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
# assert f.x == 1
dill.copy(f)
and the error:
Traceback (most recent call last):
File "aaron_test2.py", line 17, in <module>
dill.copy(f)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 114, in copy
return loads(dumps(obj))
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 147, in loads
return load(file)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 137, in load
obj = pik.load()
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 849, in load
dispatch[key[0]](self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 1094, in load_newobj
obj = cls.__new__(cls, *args)
RuntimeError: uninitialized staticmethod object
from dill.
Here's a little further exploration:
# calling dill.dumps(f)
T2: <class '__main__.Foo'>
F2: <function _create_type at 0x109eea320>
T1: <class 'type'>
F2: <function _load_type at 0x109eea290>
T1: <class 'object'>
D2: <dict object at 0x1096b2290>
T1: <class 'staticmethod'>
F1: <function Foo.bar at 0x1096af950>
F2: <function _create_function at 0x109eea3b0>
Co: <code object bar at 0x109473ae0, file "aaron_test2.py", line 10>
F2: <function _unmarshal at 0x109eea200>
D1: <dict object at 0x1094396c8>
D2: <dict object at 0x1095772d8>
b'\x80\x03cdill.dill\n_create_type\nq\x00(cdill.dill\n_load_type\nq\x01X\x04\x00\x00\x00typeq\x02\x85q\x03Rq\x04X\x03\x00\x00\x00Fooq\x05h\x01X\x06\x00\x00\x00objectq\x06\x85q\x07Rq\x08\x85q\t}q\n(X\x07\x00\x00\x00__doc__q\x0bNX\x07\x00\x00\x00__new__q\x0ch\x01X\x0c\x00\x00\x00staticmethodq\r\x85q\x0eRq\x0f)\x81q\x10X\x03\x00\x00\x00barq\x11cdill.dill\n_create_function\nq\x12(cdill.dill\n_unmarshal\nq\x13C\x8bc\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x03\x00\x00\x00C\x00\x00\x00s\x16\x00\x00\x00|\x00\x00j\x00\x00|\x00\x00j\x01\x00|\x00\x00j\x02\x00f\x03\x00S(\x01\x00\x00\x00N(\x03\x00\x00\x00u\x01\x00\x00\x00xu\x01\x00\x00\x00yu\x01\x00\x00\x00z(\x01\x00\x00\x00u\x04\x00\x00\x00self(\x00\x00\x00\x00(\x00\x00\x00\x00u\x0e\x00\x00\x00aaron_test2.pyu\x03\x00\x00\x00bar\n\x00\x00\x00s\x02\x00\x00\x00\x00\x01q\x14\x85q\x15Rq\x16c__builtin__\n__main__\nh\x11NN}q\x17tq\x18Rq\x19X\r\x00\x00\x00__slotnames__q\x1a]q\x1bX\n\x00\x00\x00__module__q\x1cX\x08\x00\x00\x00__main__q\x1dutq\x1eRq\x1f)\x81q .'
# calling dill.dumps(f, byref=True)
T5: <class '__main__.Foo'>
b'\x80\x03c__main__\nFoo\nq\x00)\x81q\x01.'
from dill.
Interesting...
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
obj.x, obj.y, obj.z = x,y,z
return obj
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
assert f.x == 1
dill.detect.trace(True)
dill.copy(f, byref=True)
Yields:
T5: <class '__main__.Foo'>
D2: <dict object at 0x10868dab8>
Traceback (most recent call last):
File "aaron_test2.py", line 17, in <module>
dill.copy(f, byref=True)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 114, in copy
return loads(dumps(obj, *args, **kwds))
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 147, in loads
return load(file)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 137, in load
obj = pik.load()
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 849, in load
dispatch[key[0]](self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 1094, in load_newobj
obj = cls.__new__(cls, *args)
TypeError: __new__() missing 3 required positional arguments: 'x', 'y', and 'z'
from dill.
apparently, this is expected behavior…
http://bugs.python.org/issue18359
https://docs.python.org/3/library/pickle.html#object.__getnewargs__
https://docs.python.org/2/library/pickle.html#object.__getnewargs__
Also, __getinitargs__
for old-style classes, and __getnewargs_ex__
for protocol=4
So, modifying the code…
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
obj.x, obj.y, obj.z = x,y,z
return obj
def __getnewargs__(self):
return self.x, self.y, self.z
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
assert f.x == 1
dill.detect.trace(True)
dill.copy(f, byref=True)
Works.
T5: <class '__main__.Foo'>
D2: <dict object at 0x102209ab8>
from dill.
although, when byref=False
, it still fails with the same RuntimeError
Now, for the original test case from @asmeurer, with slight modification to use __getnewargs__
:
class _sentinel:
pass
import dill
class Test:
def __new__(cls, name):
if name == _sentinel:
obj = object.__new__(cls)
else:
obj = type(name.capitalize() + cls.__name__, (cls,), {})(_sentinel)
obj.name = name
return obj
def __getnewargs__(self):
return (self.name,)
t = Test("Aaron")
_t = dill.dumps(t, byref=True)
pt = dill.loads(_t)
and when it's run, the traceback is…
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 689, in save_global
klass = getattr(mod, name)
AttributeError: 'module' object has no attribute 'AaronTest'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "aaron_test.py", line 20, in <module>
_t = dill.dumps(t, byref=True)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 130, in dumps
dump(obj, file, protocol, byref)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 123, in dump
pik.dump(obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 235, in dump
self.save(obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 342, in save
self.save_reduce(obj=obj, *rv)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 407, in save_reduce
save(cls)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 297, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 757, in save_type
StockPickler.save_global(pickler, obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 693, in save_global
(obj, module, name))
_pickle.PicklingError: Can't pickle <class '__main__.AaronTest'>: it's not found as __main__.AaronTest
which, I believe is simply a matter of pickle's save_global
trying to find a function whose name is pointing to the wrong location. Apparently, dill.Pickler.save_global
can pass a keyword name
to save_global
from the pickle
module to point an oddly named function to the right location, however name
is currently not used in dill
.
from dill.
Hi Mike, not sure if this is the same issue?
When pickling an instance of a numpy.ndarray subclass, attributes disappear:
import numpy as np
import dill
class TestArray(np.ndarray):
def __new__(cls, input_array, color):
obj = np.asarray(input_array).view(cls)
obj.color = color
return obj
def __array_finalize__(self, obj):
if obj is None:
return
if isinstance(obj, type(self)):
self.color = obj.color
def __getnewargs__(self):
return np.asarray(self), self.color
a1 = TestArray(np.zeros(100), color='green')
print(a1.__dict__)
print(dill.copy(a1).__dict__)
a2 = a1[0:9]
print(a2.__dict__)
print(dill.copy(a2).__dict__)
output:
{'color': 'green'}
{}
{'color': 'green'}
{}
from dill.
This doesn't help: print(dill.copy(a1, byref=True).__dict__)
so it might also be something else. I'll have a look at it with dill.detect.trace(True)
from dill.
this looks like it might be a separate issue.
from dill.
RESTATING: I believe this is simply a matter of pickle's save_global
trying to find a function whose name is pointing to the wrong location. Apparently, dill.Pickler.save_global
can pass a keyword name
to save_global
from the pickle
module to point an oddly named function to the right location, however name is currently not used in dill
.
I believe this should cover a lot of the remaining objects that dill
fails on.
from dill.
Something like this hack may be an option:
# file: m.py
class A:
class B:
pass
>>> import m
>>> import dill
>>> x = dill.dumps(compile(dill.source.importable(m.A.B, source=True), '<string>', 'exec'))
>>> exec(dill.loads(x))
>>> B
<class __main__.B at 0x10860c9a8>
from dill.
Related Issues (20)
- tqdm is not serializable? HOT 4
- Class attributes identical to inherited ones are wrongly deleted HOT 3
- Problem serializing instance of a class that uses a module HOT 4
- Error trying to serialize a binary file handle HOT 3
- Cannot pickle TFAutomodels HOT 1
- dill.load - object has no attribute in custom class HOT 2
- Python 3.12: ModuleNotFoundError: No module named 'setuptools' HOT 5
- How to ensure the same functions serialize to the same bytes?
- Cannot use callable that was pickled within pytest HOT 14
- dill.source.getsource applied on lambda function returns many cached lines
- Class attributes changes at load if the object is in a list or tuple. HOT 1
- nan type drift for np.nan HOT 3
- dill.source.findsource fails when in asyncio REPL
- `save_function()` can't save function in a submodule that has the same name as an attribute of the parent module HOT 2
- bytes length not a multiple of item size
- dill.source.getsource returns decorated function instead of function HOT 2
- 0.3.7 incorrectly pickles the class definition for module/class with the same name HOT 7
- Exception: --- Logging error --- HOT 3
- Use dump_module() in jupyter notebook with remote kernel HOT 1
- UnpicklingError using dill but not stdlib pickle HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from dill.