Comments (10)
mmckerns
commented on August 17, 2024
Here's a simple case:
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
# obj.x, obj.y, obj.z = x,y,z
return obj
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
# assert f.x == 1
dill.copy(f)
and the error:
Traceback (most recent call last):
File "aaron_test2.py", line 17, in <module>
dill.copy(f)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 114, in copy
return loads(dumps(obj))
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 147, in loads
return load(file)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 137, in load
obj = pik.load()
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 849, in load
dispatch[key[0]](self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 1094, in load_newobj
obj = cls.__new__(cls, *args)
RuntimeError: uninitialized staticmethod object
from dill.
mmckerns
commented on August 17, 2024
Here's a little further exploration:
# calling dill.dumps(f)
T2: <class '__main__.Foo'>
F2: <function _create_type at 0x109eea320>
T1: <class 'type'>
F2: <function _load_type at 0x109eea290>
T1: <class 'object'>
D2: <dict object at 0x1096b2290>
T1: <class 'staticmethod'>
F1: <function Foo.bar at 0x1096af950>
F2: <function _create_function at 0x109eea3b0>
Co: <code object bar at 0x109473ae0, file "aaron_test2.py", line 10>
F2: <function _unmarshal at 0x109eea200>
D1: <dict object at 0x1094396c8>
D2: <dict object at 0x1095772d8>
b'\x80\x03cdill.dill\n_create_type\nq\x00(cdill.dill\n_load_type\nq\x01X\x04\x00\x00\x00typeq\x02\x85q\x03Rq\x04X\x03\x00\x00\x00Fooq\x05h\x01X\x06\x00\x00\x00objectq\x06\x85q\x07Rq\x08\x85q\t}q\n(X\x07\x00\x00\x00__doc__q\x0bNX\x07\x00\x00\x00__new__q\x0ch\x01X\x0c\x00\x00\x00staticmethodq\r\x85q\x0eRq\x0f)\x81q\x10X\x03\x00\x00\x00barq\x11cdill.dill\n_create_function\nq\x12(cdill.dill\n_unmarshal\nq\x13C\x8bc\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x03\x00\x00\x00C\x00\x00\x00s\x16\x00\x00\x00|\x00\x00j\x00\x00|\x00\x00j\x01\x00|\x00\x00j\x02\x00f\x03\x00S(\x01\x00\x00\x00N(\x03\x00\x00\x00u\x01\x00\x00\x00xu\x01\x00\x00\x00yu\x01\x00\x00\x00z(\x01\x00\x00\x00u\x04\x00\x00\x00self(\x00\x00\x00\x00(\x00\x00\x00\x00u\x0e\x00\x00\x00aaron_test2.pyu\x03\x00\x00\x00bar\n\x00\x00\x00s\x02\x00\x00\x00\x00\x01q\x14\x85q\x15Rq\x16c__builtin__\n__main__\nh\x11NN}q\x17tq\x18Rq\x19X\r\x00\x00\x00__slotnames__q\x1a]q\x1bX\n\x00\x00\x00__module__q\x1cX\x08\x00\x00\x00__main__q\x1dutq\x1eRq\x1f)\x81q .'
# calling dill.dumps(f, byref=True)
T5: <class '__main__.Foo'>
b'\x80\x03c__main__\nFoo\nq\x00)\x81q\x01.'
from dill.
mmckerns
commented on August 17, 2024
Interesting...
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
obj.x, obj.y, obj.z = x,y,z
return obj
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
assert f.x == 1
dill.detect.trace(True)
dill.copy(f, byref=True)
Yields:
T5: <class '__main__.Foo'>
D2: <dict object at 0x10868dab8>
Traceback (most recent call last):
File "aaron_test2.py", line 17, in <module>
dill.copy(f, byref=True)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 114, in copy
return loads(dumps(obj, *args, **kwds))
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 147, in loads
return load(file)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 137, in load
obj = pik.load()
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 849, in load
dispatch[key[0]](self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 1094, in load_newobj
obj = cls.__new__(cls, *args)
TypeError: __new__() missing 3 required positional arguments: 'x', 'y', and 'z'
from dill.
mmckerns
commented on August 17, 2024
apparently, this is expected behavior…
http://bugs.python.org/issue18359
https://docs.python.org/3/library/pickle.html#object.__getnewargs__
https://docs.python.org/2/library/pickle.html#object.__getnewargs__
Also, __getinitargs__ for old-style classes, and __getnewargs_ex__ for protocol=4
So, modifying the code…
import dill
class Foo(object):
def __new__(cls, x, y, z):
obj = object.__new__(cls)
obj.x, obj.y, obj.z = x,y,z
return obj
def __getnewargs__(self):
return self.x, self.y, self.z
# def __init__(self, x, y, z):
# self.x, self.y, self.z = x,y,z
def bar(self):
return self.x, self.y, self.z
f = Foo(1,2,3)
assert f.x == 1
dill.detect.trace(True)
dill.copy(f, byref=True)
Works.
T5: <class '__main__.Foo'>
D2: <dict object at 0x102209ab8>
from dill.
mmckerns
commented on August 17, 2024
although, when byref=False, it still fails with the same RuntimeError
Now, for the original test case from @asmeurer, with slight modification to use __getnewargs__:
class _sentinel:
pass
import dill
class Test:
def __new__(cls, name):
if name == _sentinel:
obj = object.__new__(cls)
else:
obj = type(name.capitalize() + cls.__name__, (cls,), {})(_sentinel)
obj.name = name
return obj
def __getnewargs__(self):
return (self.name,)
t = Test("Aaron")
_t = dill.dumps(t, byref=True)
pt = dill.loads(_t)
and when it's run, the traceback is…
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 689, in save_global
klass = getattr(mod, name)
AttributeError: 'module' object has no attribute 'AaronTest'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "aaron_test.py", line 20, in <module>
_t = dill.dumps(t, byref=True)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 130, in dumps
dump(obj, file, protocol, byref)
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 123, in dump
pik.dump(obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 235, in dump
self.save(obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 342, in save
self.save_reduce(obj=obj, *rv)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 407, in save_reduce
save(cls)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 297, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/mmckerns/lib/python3.3/site-packages/dill-0.2b2.dev-py3.3.egg/dill/dill.py", line 757, in save_type
StockPickler.save_global(pickler, obj)
File "/opt/local/Library/Frameworks/Python.framework/Versions/3.3/lib/python3.3/pickle.py", line 693, in save_global
(obj, module, name))
_pickle.PicklingError: Can't pickle <class '__main__.AaronTest'>: it's not found as __main__.AaronTest
which, I believe is simply a matter of pickle's save_global trying to find a function whose name is pointing to the wrong location. Apparently, dill.Pickler.save_global can pass a keyword name to save_global from the pickle module to point an oddly named function to the right location, however name is currently not used in dill.
from dill.
mattja
commented on August 17, 2024
Hi Mike, not sure if this is the same issue?
When pickling an instance of a numpy.ndarray subclass, attributes disappear:
import numpy as np
import dill
class TestArray(np.ndarray):
def __new__(cls, input_array, color):
obj = np.asarray(input_array).view(cls)
obj.color = color
return obj
def __array_finalize__(self, obj):
if obj is None:
return
if isinstance(obj, type(self)):
self.color = obj.color
def __getnewargs__(self):
return np.asarray(self), self.color
a1 = TestArray(np.zeros(100), color='green')
print(a1.__dict__)
print(dill.copy(a1).__dict__)
a2 = a1[0:9]
print(a2.__dict__)
print(dill.copy(a2).__dict__)
output:
{'color': 'green'}
{}
{'color': 'green'}
{}
from dill.
mmckerns
commented on August 17, 2024
This doesn't help: print(dill.copy(a1, byref=True).__dict__)
so it might also be something else. I'll have a look at it with dill.detect.trace(True)
from dill.
mmckerns
commented on August 17, 2024
this looks like it might be a separate issue.
from dill.
mmckerns
commented on August 17, 2024
RESTATING: I believe this is simply a matter of pickle's save_global trying to find a function whose name is pointing to the wrong location. Apparently, dill.Pickler.save_global can pass a keyword name to save_global from the pickle module to point an oddly named function to the right location, however name is currently not used in dill.
I believe this should cover a lot of the remaining objects that dill fails on.
from dill.
mmckerns
commented on August 17, 2024
Something like this hack may be an option:
# file: m.py
class A:
class B:
pass
>>> import m
>>> import dill
>>> x = dill.dumps(compile(dill.source.importable(m.A.B, source=True), '<string>', 'exec'))
>>> exec(dill.loads(x))
>>> B
<class __main__.B at 0x10860c9a8>
from dill.
Related Issues (20)
- tqdm is not serializable? HOT 4
- Class attributes identical to inherited ones are wrongly deleted HOT 3
- Problem serializing instance of a class that uses a module HOT 4
- Error trying to serialize a binary file handle HOT 3
- Cannot pickle TFAutomodels HOT 1
- dill.load - object has no attribute in custom class HOT 2
- Python 3.12: ModuleNotFoundError: No module named 'setuptools' HOT 5
- How to ensure the same functions serialize to the same bytes?
- Cannot use callable that was pickled within pytest HOT 14
- dill.source.getsource applied on lambda function returns many cached lines
- Class attributes changes at load if the object is in a list or tuple. HOT 1
- nan type drift for np.nan HOT 3
- dill.source.findsource fails when in asyncio REPL
- `save_function()` can't save function in a submodule that has the same name as an attribute of the parent module HOT 2
- bytes length not a multiple of item size
- dill.source.getsource returns decorated function instead of function HOT 2
- 0.3.7 incorrectly pickles the class definition for module/class with the same name HOT 7
- Exception: --- Logging error --- HOT 3
- Use dump_module() in jupyter notebook with remote kernel HOT 1
- UnpicklingError using dill but not stdlib pickle HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from dill.