def makePickle(self, record):
"""
Pickles the record in binary format with a length prefix, and
returns it ready for transmission across the socket.
"""
ei = record.exc_info
if ei:
dummy = self.format(record) # just to get traceback text into record.exc_text
record.exc_info = None # to avoid Unpickleable error
s = cPickle.dumps(record.__dict__, 1)
if ei:
record.exc_info = ei # for next handler
slen = struct.pack(">L", len(s))
return slen + s
python类dumps()的实例源码
def value_encode(self, val):
return val, _quote( dumps(val) )
# end SerialCookie
def value_encode(self, val):
if type(val) == type(""):
return val, _quote(val)
else:
return val, _quote( dumps(val) )
# end SmartCookie
###########################################################
# Backwards Compatibility: Don't break any existing code!
# We provide Cookie() as an alias for SmartCookie()
def fetch(self, spider):
while True:
item_dict = spider.fetch()
item = []
for attr_name in spider.attr:
item.append(item_dict[attr_name])
while self.redis.llen(config.solver_prototxt) >= self.max_cache_item_num:
time.sleep(self.wait_time)
self.redis.rpush(config.solver_prototxt, cPickle.dumps(item, cPickle.HIGHEST_PROTOCOL))
def pack(self, item):
if isinstance(item, np.ndarray):
return item
else:
return 'OBJ_' + cPickle.dumps(item, -1)
def picklechops(chops):
"""Pickles and base64encodes it's argument chops"""
value = zlib.compress(dumps(chops))
encoded = base64.encodestring(value)
return encoded.strip()
def testPackPickleUnpack(self):
for TEST_VALUES in self.TEST_VALUES:
x = self.Struct(**{k:v for k,v in TEST_VALUES.iteritems()})
pschema = cPickle.loads(cPickle.dumps(self.schema))
dx = pschema.unpack(self.schema.pack(x))
for k,v in TEST_VALUES.iteritems():
self.assertTrue(hasattr(dx, k))
self.assertEqual(getattr(dx, k), v)
for k in self.Struct.__slots__:
if k not in TEST_VALUES:
self.assertFalse(hasattr(dx, k))
def _handle_serialization(func):
def wrapped(session, params):
params = pickle.loads(params['params'])
rv = func(session, *params['args'], **params['kwargs'])
return pickle.dumps(rv)
return wrapped
def call_plugin_serialized(self, plugin, fn, *args, **kwargs):
params = {'params': pickle.dumps(dict(args=args, kwargs=kwargs))}
rv = self.call_plugin(plugin, fn, params)
return pickle.loads(rv)
def set(self, key, value, timeout=None):
expires = self._get_expiration(timeout)
self._prune()
self._cache[key] = (expires, pickle.dumps(value,
pickle.HIGHEST_PROTOCOL))
return True
def add(self, key, value, timeout=None):
expires = self._get_expiration(timeout)
self._prune()
item = (expires, pickle.dumps(value,
pickle.HIGHEST_PROTOCOL))
if key in self._cache:
return False
self._cache.setdefault(key, item)
return True
def dump_object(self, value):
"""Dumps an object into a string for redis. By default it serializes
integers as regular string and pickle dumps everything else.
"""
t = type(value)
if t in integer_types:
return str(value).encode('ascii')
return b'!' + pickle.dumps(value)
def _encode_request(self, request):
"""Encode a request object"""
return pickle.dumps(request_to_dict(request, self.spider), protocol=-1)
def send(self, obj):
'Send one object.'
string = _base255.encode(_cPickle.dumps(obj, _cPickle.HIGHEST_PROTOCOL)) + '\0'
self.__send.acquire()
try:
self.__sock.sendall(string)
finally:
self.__send.release()
def __call__(self, *args):
import cPickle
str = cPickle.dumps(args)
if not self.memo.has_key(str):
self.memo[str] = self.fn(*args)
return self.memo[str]
def put(self,msg):
if self.is_open():
data = cPickle.dumps(msg,1)
self.out.write("%d\n" % len(data))
self.out.write(data)
self.out.flush()
else:
raise Exception("Pipe closed")
def store(self):
"""
Store the data for next runs, sets the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`. Uses a temporary
file to avoid problems on ctrl+c.
"""
data = {}
for x in SAVED_ATTRS:
data[x] = getattr(self, x)
db = os.path.join(self.variant_dir, Context.DBFILE)
try:
waflib.Node.pickle_lock.acquire()
waflib.Node.Nod3 = self.node_class
x = cPickle.dumps(data, PROTOCOL)
finally:
waflib.Node.pickle_lock.release()
Utils.writef(db + '.tmp', x, m='wb')
try:
st = os.stat(db)
os.remove(db)
if not Utils.is_win32: # win32 has no chown but we're paranoid
os.chown(db + '.tmp', st.st_uid, st.st_gid)
except (AttributeError, OSError):
pass
# do not use shutil.move (copy is not thread-safe)
os.rename(db + '.tmp', db)
def send_response(conn, ret, out, err, exc):
if out or err or exc:
data = (out, err, exc)
data = cPickle.dumps(data, -1)
else:
data = ''
params = [RES, str(ret), str(len(data))]
# no need for the cookie in the response
conn.send(make_header(params))
if data:
conn.send(data)
def send_response(self, ret, out, err, exc):
if out or err or exc:
data = (out, err, exc)
data = cPickle.dumps(data, -1)
else:
data = ''
params = [RES, str(ret), str(len(data))]
# no need for the cookie in the response
self.wfile.write(make_header(params))
if data:
self.wfile.write(data)
self.wfile.flush()
def store(self):
"""
Store the data for next runs, sets the attributes listed in :py:const:`waflib.Build.SAVED_ATTRS`. Uses a temporary
file to avoid problems on ctrl+c.
"""
data = {}
for x in SAVED_ATTRS:
data[x] = getattr(self, x)
db = os.path.join(self.variant_dir, Context.DBFILE)
try:
waflib.Node.pickle_lock.acquire()
waflib.Node.Nod3 = self.node_class
x = cPickle.dumps(data, PROTOCOL)
finally:
waflib.Node.pickle_lock.release()
Utils.writef(db + '.tmp', x, m='wb')
try:
st = os.stat(db)
os.remove(db)
if not Utils.is_win32: # win32 has no chown but we're paranoid
os.chown(db + '.tmp', st.st_uid, st.st_gid)
except (AttributeError, OSError):
pass
# do not use shutil.move (copy is not thread-safe)
os.rename(db + '.tmp', db)
