def loads_with_persistent_ids(str, env):
"""
Performs a pickle loads on the given string, substituting the given
TradingEnvironment in to any tokenized representations of a
TradingEnvironment or AssetFinder.
Parameters
----------
str : String
The string representation of the object to be unpickled.
env : TradingEnvironment
The TradingEnvironment to be inserted to the unpickled object.
Returns
-------
obj
An unpickled object formed from the parameter 'str'.
"""
file = BytesIO(str)
unpickler = pickle.Unpickler(file)
unpickler.persistent_load = partial(_persistent_load, env=env)
return unpickler.load()
python类Unpickler()的实例源码
def loads_with_persistent_ids(str, env):
"""
Performs a pickle loads on the given string, substituting the given
TradingEnvironment in to any tokenized representations of a
TradingEnvironment or AssetFinder.
Parameters
----------
str : String
The string representation of the object to be unpickled.
env : TradingEnvironment
The TradingEnvironment to be inserted to the unpickled object.
Returns
-------
obj
An unpickled object formed from the parameter 'str'.
"""
file = BytesIO(str)
unpickler = pickle.Unpickler(file)
unpickler.persistent_load = partial(_persistent_load, env=env)
return unpickler.load()
def loads(self, buf):
class PersUnpickler(pickle.Unpickler):
def persistent_load(subself, obj):
return self.persistent_load(obj)
f = StringIO(buf)
u = PersUnpickler(f)
return u.load()
def _vrecv(self, id):
self._flush()
if self._replies.has_key(id):
if self._verbose > 1: print "retrieving previous reply, id = %d" % id
reply = self._replies[id]
del self._replies[id]
return reply
aid = abs(id)
while 1:
if self._verbose > 1: print "waiting for reply, id = %d" % id
rp = pickle.Unpickler(self._rf)
reply = rp.load()
del rp
if self._verbose > 1: print "got reply: %s" % repr(reply)
rid = reply[2]
arid = abs(rid)
if arid == aid:
if self._verbose > 1: print "got it"
return reply
self._replies[rid] = reply
if arid > aid:
if self._verbose > 1: print "got higher id, assume all ok"
return (None, None, id)
def _dorequest(self, rf, wf):
rp = pickle.Unpickler(rf)
try:
request = rp.load()
except EOFError:
return 0
if self._verbose > 1: print "Got request: %s" % repr(request)
try:
methodname, args, id = request
if '.' in methodname:
reply = (None, self._special(methodname, args), id)
elif methodname[0] == '_':
raise NameError, "illegal method name %s" % repr(methodname)
else:
method = getattr(self, methodname)
reply = (None, apply(method, args), id)
except:
reply = (sys.exc_type, sys.exc_value, id)
if id < 0 and reply[:2] == (None, None):
if self._verbose > 1: print "Suppress reply"
return 1
if self._verbose > 1: print "Send reply: %s" % repr(reply)
wp = pickle.Pickler(wf)
wp.dump(reply)
return 1
def find_class(self, module, name):
"""
Overridden from the original 'Unpickler' class. Needed to rebuild PyMod object which have
complex modules names. 'Unpickler' rebuilds objects using the 'fully qualified' name
reference of their classes (the class name is pickled, along with the name of the module the
class is defined in). Since PyMOL plugin modules may yield different 'fully qualified' names
depending on the system, PyMod objects are rebuilt using only the name of their classes.
"""
try:
# Try the standard routine of pickle.
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
return klass
except:
# Build object by class name.
try:
name = name.rstrip("\n\r") # Needed to fix some old Windows versions behaviour.
except:
pass
return globals()[name]
def load_build(self):
"""Set the state of a newly created object.
We capture it to replace our place-holder objects,
NDArrayWrapper, by the array we are interested in. We
replace them directly in the stack of pickler.
"""
Unpickler.load_build(self)
if isinstance(self.stack[-1], NDArrayWrapper):
if self.np is None:
raise ImportError("Trying to unpickle an ndarray, "
"but numpy didn't import correctly")
nd_array_wrapper = self.stack.pop()
array = nd_array_wrapper.read(self)
self.stack.append(array)
# Be careful to register our new method.
def python_memcache_deserializer(key, value, flags):
if flags == 0:
return value
if flags & FLAG_INTEGER:
return int(value)
if flags & FLAG_LONG:
return long(value)
if flags & FLAG_PICKLE:
try:
buf = StringIO(value)
unpickler = pickle.Unpickler(buf)
return unpickler.load()
except Exception:
logging.info('Pickle error', exc_info=True)
return None
return value
def load_build(self):
""" This method is called to set the state of a newly created
object.
We capture it to replace our place-holder objects,
NDArrayWrapper, by the array we are interested in. We
replace them directly in the stack of pickler.
"""
Unpickler.load_build(self)
if isinstance(self.stack[-1], NDArrayWrapper):
if self.np is None:
raise ImportError('Trying to unpickle an ndarray, '
"but numpy didn't import correctly")
nd_array_wrapper = self.stack.pop()
array = nd_array_wrapper.read(self)
self.stack.append(array)
# Be careful to register our new method.
def load(f, persistent_load=PersistentNdarrayLoad):
"""Load a file that was dumped to a zip file.
:param f: The file handle to the zip file to load the object from.
:type f: file
:param persistent_load: The persistent loading function to use for
unpickling. This must be compatible with the `persisten_id` function
used when pickling.
:type persistent_load: callable, optional
.. versionadded:: 0.8
"""
with closing(zipfile.ZipFile(f, 'r')) as zip_file:
p = pickle.Unpickler(BytesIO(zip_file.open('pkl').read()))
p.persistent_load = persistent_load(zip_file)
return p.load()
def load_build(self):
""" This method is called to set the state of a newly created
object.
We capture it to replace our place-holder objects,
NDArrayWrapper, by the array we are interested in. We
replace them directly in the stack of pickler.
"""
Unpickler.load_build(self)
if isinstance(self.stack[-1], NDArrayWrapper):
if self.np is None:
raise ImportError('Trying to unpickle an ndarray, '
"but numpy didn't import correctly")
nd_array_wrapper = self.stack.pop()
array = nd_array_wrapper.read(self)
self.stack.append(array)
# Be careful to register our new method.
def __getitem__(self, key):
try:
value = self.cache[key]
except KeyError:
f = StringIO(self.dict[key])
value = Unpickler(f).load()
if self.writeback:
self.cache[key] = value
return value
def set_location(self, key):
(key, value) = self.dict.set_location(key)
f = StringIO(value)
return (key, Unpickler(f).load())
def next(self):
(key, value) = self.dict.next()
f = StringIO(value)
return (key, Unpickler(f).load())
def previous(self):
(key, value) = self.dict.previous()
f = StringIO(value)
return (key, Unpickler(f).load())
def first(self):
(key, value) = self.dict.first()
f = StringIO(value)
return (key, Unpickler(f).load())
def find_class(self, module, cname):
# Pickle tries to load a couple things like copy_reg and
# __builtin__.object even though a pickle file doesn't
# explicitly reference them (afaict): allow them to be loaded
# normally.
if module in ('copy_reg', '__builtin__'):
thing = pickle.Unpickler.find_class(self, module, cname)
return thing
return makeFakeClass(module, cname)
def load(self):
result = pickle.Unpickler.load(self)
self.extended_init = pickle.Unpickler.load(self)
return result
def ResourceUnpickler(path, registry = None):
fl = open(path)
result = Unpickler(fl).load()
return result
def __getitem__(self, key):
try:
value = self.cache[key]
except KeyError:
f = StringIO(self.dict[key])
value = Unpickler(f).load()
if self.writeback:
self.cache[key] = value
return value
def set_location(self, key):
(key, value) = self.dict.set_location(key)
f = StringIO(value)
return (key, Unpickler(f).load())
def next(self):
(key, value) = self.dict.next()
f = StringIO(value)
return (key, Unpickler(f).load())
def previous(self):
(key, value) = self.dict.previous()
f = StringIO(value)
return (key, Unpickler(f).load())
def last(self):
(key, value) = self.dict.last()
f = StringIO(value)
return (key, Unpickler(f).load())
def __init__(self, f, objmap, shortcuts=None):
pickle.Unpickler.__init__(self, f)
if shortcuts:
objmap = dict((k % shortcuts, v % shortcuts)
for k, v in objmap.items())
self._objmap = objmap
def __reduce__(self):
return (spickle._reconstructor, (pickle.Unpickler, str, StringIO.StringIO(
"csubprocess\nPopen\np0\n((S'/bin/ls'\np1\nS'/tmp'\np2\ntp3\ntp4\nRp5.")))
def load(filename):
"Load in an existing sequence from disk"
if filename.endswith('.gz'):
fp = gzip.GzipFile(filename, 'r')
elif filename.endswith('.bz2'):
fp = bz2.BZ2File(filename, 'r')
else:
fp = open(filename, 'rb')
u = Unpickler(fp)
return MonkeySequence(data=u.load())
def load(filename):
'''
Load a saved monkeytree from disk
'''
if filename.endswith('.gz'):
fp = gzip.GzipFile(filename, 'r')
elif filename.endswith('.bz2'):
fp = bz2.BZ2File(filename, 'r')
else:
fp = open(filename, 'rb')
u = Unpickler(fp)
return MonkeyTree(root=u.load())
def main(argv=None):
"""
Get the mapping for the target recipe.
"""
if len(argv) != 1:
print("Error, need one argument!", file=sys.stderr)
return 2
cachefile = argv[0]
with open(cachefile, "rb") as cachefile:
pickled = pickle.Unpickler(cachefile)
while cachefile:
try:
key = pickled.load()
val = pickled.load()
except Exception:
break
if isinstance(val, CoreRecipeInfo) and (not val.skipped):
pn = val.pn
# Filter out the native recipes.
if key.startswith('virtual:native:') or pn.endswith("-native"):
continue
# 1.0 is the default version for a no PV recipe.
if "pv" in val.__dict__:
pv = val.pv
else:
pv = "1.0"
print("%s %s %s %s" % (key, pn, pv, ' '.join(val.packages)))
def init_cache(self, d, cache_file_name=None):
cachedir = (d.getVar("PERSISTENT_DIR", True) or
d.getVar("CACHE", True))
if cachedir in [None, '']:
return
bb.utils.mkdirhier(cachedir)
self.cachefile = os.path.join(cachedir,
cache_file_name or self.__class__.cache_file_name)
logger.debug(1, "Using cache in '%s'", self.cachefile)
glf = bb.utils.lockfile(self.cachefile + ".lock")
try:
with open(self.cachefile, "rb") as f:
p = pickle.Unpickler(f)
data, version = p.load()
except:
bb.utils.unlockfile(glf)
return
bb.utils.unlockfile(glf)
if version != self.__class__.CACHE_VERSION:
return
self.cachedata = data
