def waitfinish(self, waiter=os.waitpid):
pid, systemstatus = waiter(self.pid, 0)
if systemstatus:
if os.WIFSIGNALED(systemstatus):
exitstatus = os.WTERMSIG(systemstatus) + 128
else:
exitstatus = os.WEXITSTATUS(systemstatus)
else:
exitstatus = 0
signal = systemstatus & 0x7f
if not exitstatus and not signal:
retval = self.RETVAL.open('rb')
try:
retval_data = retval.read()
finally:
retval.close()
retval = marshal.loads(retval_data)
else:
retval = None
stdout = self.STDOUT.read()
stderr = self.STDERR.read()
self._removetemp()
return Result(exitstatus, signal, retval, stdout, stderr)
python类loads()的实例源码
def geocode(api, params, l4):
params['autocomplete']=0
params['q'] = params['q'].strip()
try:
r = requests.get(api, params)
j = json.loads(r.text)
global geocode_count
geocode_count += 1
if 'features' in j and len(j['features'])>0:
j['features'][0]['l4'] = l4
return(j['features'][0])
else:
return(None)
except:
print("err_geocode", params, l4)
return(None)
def geocode(api, params, l4):
params['autocomplete']=0
params['q'] = params['q'].strip()
try:
r = requests.get(api, params)
j = json.loads(r.text)
global geocode_count
geocode_count += 1
if 'features' in j and len(j['features'])>0:
j['features'][0]['l4'] = l4
return(j['features'][0])
else:
return(None)
except:
print("err_geocode", params, l4)
return(None)
def geocode(api, params, l4):
params['autocomplete']=0
params['q'] = params['q'].strip()
try:
r = requests.get(api, params)
j = json.loads(r.text)
global geocode_count
geocode_count += 1
if 'features' in j and len(j['features'])>0:
j['features'][0]['l4'] = l4
return(j['features'][0])
else:
return(None)
except:
print(json.dumps({'action':'erreur','params': params, 'l4': l4}))
return(None)
def sw_compute_features(learn_data, overwrite_existing=False, worker_id=None):
# learn_data = db['learns'].find_one(learn_id)
model_data = db[learn_data['Model'][-1]].find_one(learn_data['Model'][0])
# sample_df = load_df_from_sample_notation(model_data['Initial Sample Location'])
if not check_sample_exists(model_data['Feature Sample Location']) or overwrite_existing:
feature_generating_function_code = marshal.loads(db[model_data['Feature Generation Function'][-1]]\
.find_one(model_data['Feature Generation Function'][0])['Code'])
feature_generating_function = types.FunctionType(feature_generating_function_code, globals())
# save_df_to_sample_notation(, model_data['Feature Sample Location'])
learn_data = feature_generating_function(learn_data, model_data)
learn_data['Status']['Features Computed'] = True
db['learns'].update(learn_data['_id'], learn_data)
def _remoteInvocationMobileCode(self, method, flags, *args):
# special trimmed-down version for mobile code methods (no locking etc)
body=pickle.dumps((self.URI.objectID,method,flags,args),Pyro.config.PYRO_PICKLE_FORMAT)
sock_sendmsg(self.conn.sock, self.createMsg(body), self.timeout)
ver,answer,pflags = self.receiveMsg(self.conn,1)
if answer is None:
raise ProtocolError('incorrect answer received')
answer=pickle.loads(answer)
if isinstance(answer,PyroExceptionCapsule):
if isinstance(answer.excObj,_InternalNoModuleError):
# server couldn't load module, supply it
return self.processMissingModuleError(answer.excObj, method, flags, args)
else:
# we have an encapsulated exception, raise it again.
answer.raiseEx()
return answer
def __fetch_compiled(self, url) :
import marshal
module_src = None
try :
module_compiled = urlopen(url + 'c').read() # from blah.py --> blah.pyc
try :
module_src = marshal.loads(module_compiled[8:]) # Strip the .pyc file header of Python up to 3.3
return module_src
except ValueError :
pass
try :
module_src = marshal.loads(module_compiled[12:])# Strip the .pyc file header of Python 3.3 and onwards (changed .pyc spec)
return module_src
except ValueError :
pass
except IOError as e:
logger.debug("[-] No compiled version ('.pyc') for '%s' module found!" % url.split('/')[-1])
return module_src
def test_int64(self):
# Simulate int marshaling on a 64-bit box. This is most interesting if
# we're running the test on a 32-bit box, of course.
def to_little_endian_string(value, nbytes):
b = bytearray()
for i in range(nbytes):
b.append(value & 0xff)
value >>= 8
return b
maxint64 = (1 << 63) - 1
minint64 = -maxint64-1
for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
while base:
s = b'I' + to_little_endian_string(base, 8)
got = marshal.loads(s)
self.assertEqual(base, got)
if base == -1: # a fixed-point for shifting right 1
base = 0
else:
base >>= 1
def test_floats(self):
# Test a few floats
small = 1e-25
n = sys.maxsize * 3.7e250
while n > small:
for expected in (-n, n):
self.helper(float(expected))
n /= 123.4567
f = 0.0
s = marshal.dumps(f, 2)
got = marshal.loads(s)
self.assertEqual(f, got)
# and with version <= 1 (floats marshalled differently then)
s = marshal.dumps(f, 1)
got = marshal.loads(s)
self.assertEqual(f, got)
n = sys.maxsize * 3.7e-250
while n < small:
for expected in (-n, n):
f = float(expected)
self.helper(f)
self.helper(f, 1)
n *= 123.4567
def test_recursion_limit(self):
# Create a deeply nested structure.
head = last = []
# The max stack depth should match the value in Python/marshal.c.
if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
MAX_MARSHAL_STACK_DEPTH = 1500
else:
MAX_MARSHAL_STACK_DEPTH = 2000
for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
last.append([0])
last = last[-1]
# Verify we don't blow out the stack with dumps/load.
data = marshal.dumps(head)
new_head = marshal.loads(data)
# Don't use == to compare objects, it can exceed the recursion limit.
self.assertEqual(len(new_head), len(head))
self.assertEqual(len(new_head[0]), len(head[0]))
self.assertEqual(len(new_head[-1]), len(head[-1]))
last.append([0])
self.assertRaises(ValueError, marshal.dumps, head)
def func_load(code, defaults=None, closure=None, globs=None):
"""Deserializes a user defined function.
# Arguments
code: bytecode of the function.
defaults: defaults of the function.
closure: closure of the function.
globs: dictionary of global objects.
# Returns
A function object.
"""
if isinstance(code, (tuple, list)): # unpack previous dump
code, defaults, closure = code
code = marshal.loads(code.encode('raw_unicode_escape'))
if globs is None:
globs = globals()
return python_types.FunctionType(code, globs,
name=code.co_name,
argdefs=defaults,
closure=closure)
def loadtoc(self):
"""
Overridable.
Default: After magic comes an int (4 byte native) giving the
position of the TOC within self.lib.
Default: The TOC is a marshal-able string.
"""
self.lib.seek(self.start + self.TOCPOS)
(offset,) = struct.unpack('!i', self.lib.read(4))
self.lib.seek(self.start + offset)
# Use marshal.loads() since load() arg must be a file object
# Convert the read list into a dict for faster access
self.toc = dict(marshal.loads(self.lib.read()))
######## This is what is called by FuncImporter #######
## Since an Archive is flat, we ignore parent and modname.
#XXX obsolete - imputil only code
## def get_code(self, parent, modname, fqname):
## pass
####### Core method - Override as needed #########
def extract(self, name):
"""
Get the object corresponding to name, or None.
For use with imputil ArchiveImporter, object is a python code object.
'name' is the name as specified in an 'import name'.
'import a.b' will become:
extract('a') (return None because 'a' is not a code object)
extract('a.__init__') (return a code object)
extract('a.b') (return a code object)
Default implementation:
self.toc is a dict
self.toc[name] is pos
self.lib has the code object marshal-ed at pos
"""
ispkg, pos = self.toc.get(name, (0, None))
if pos is None:
return None
with self.lib:
self.lib.seek(self.start + pos)
# use marshal.loads() sind load() arg must be a file object
obj = marshal.loads(self.lib.read())
return ispkg, obj
########################################################################
# Informational methods
def extract(self, name):
(typ, pos, length) = self.toc.get(name, (0, None, 0))
if pos is None:
return None
with self.lib:
self.lib.seek(self.start + pos)
obj = self.lib.read(length)
try:
if self.cipher:
obj = self.cipher.decrypt(obj)
obj = zlib.decompress(obj)
if typ in (PYZ_TYPE_MODULE, PYZ_TYPE_PKG):
obj = marshal.loads(obj)
except EOFError:
raise ImportError("PYZ entry '%s' failed to unmarshal" % name)
return typ, obj
def test_int64(self):
# Simulate int marshaling on a 64-bit box. This is most interesting if
# we're running the test on a 32-bit box, of course.
def to_little_endian_string(value, nbytes):
bytes = []
for i in range(nbytes):
bytes.append(chr(value & 0xff))
value >>= 8
return ''.join(bytes)
maxint64 = (1L << 63) - 1
minint64 = -maxint64-1
for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
while base:
s = 'I' + to_little_endian_string(base, 8)
got = marshal.loads(s)
self.assertEqual(base, got)
if base == -1: # a fixed-point for shifting right 1
base = 0
else:
base >>= 1
def test_recursion_limit(self):
# Create a deeply nested structure.
head = last = []
# The max stack depth should match the value in Python/marshal.c.
if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
MAX_MARSHAL_STACK_DEPTH = 1000
else:
MAX_MARSHAL_STACK_DEPTH = 2000
for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
last.append([0])
last = last[-1]
# Verify we don't blow out the stack with dumps/load.
data = marshal.dumps(head)
new_head = marshal.loads(data)
# Don't use == to compare objects, it can exceed the recursion limit.
self.assertEqual(len(new_head), len(head))
self.assertEqual(len(new_head[0]), len(head[0]))
self.assertEqual(len(new_head[-1]), len(head[-1]))
last.append([0])
self.assertRaises(ValueError, marshal.dumps, head)
def test_int64(self):
# Simulate int marshaling on a 64-bit box. This is most interesting if
# we're running the test on a 32-bit box, of course.
def to_little_endian_string(value, nbytes):
bytes = []
for i in range(nbytes):
bytes.append(chr(value & 0xff))
value >>= 8
return ''.join(bytes)
maxint64 = (1L << 63) - 1
minint64 = -maxint64-1
for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
while base:
s = 'I' + to_little_endian_string(base, 8)
got = marshal.loads(s)
self.assertEqual(base, got)
if base == -1: # a fixed-point for shifting right 1
base = 0
else:
base >>= 1
def test_recursion_limit(self):
# Create a deeply nested structure.
head = last = []
# The max stack depth should match the value in Python/marshal.c.
if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
MAX_MARSHAL_STACK_DEPTH = 1000
else:
MAX_MARSHAL_STACK_DEPTH = 2000
for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
last.append([0])
last = last[-1]
# Verify we don't blow out the stack with dumps/load.
data = marshal.dumps(head)
new_head = marshal.loads(data)
# Don't use == to compare objects, it can exceed the recursion limit.
self.assertEqual(len(new_head), len(head))
self.assertEqual(len(new_head[0]), len(head[0]))
self.assertEqual(len(new_head[-1]), len(head[-1]))
last.append([0])
self.assertRaises(ValueError, marshal.dumps, head)
def load_bytecode(self, f):
"""Loads bytecode from a file or file like object."""
# make sure the magic header is correct
magic = f.read(len(bc_magic))
if magic != bc_magic:
self.reset()
return
# the source code of the file changed, we need to reload
checksum = pickle.load(f)
if self.checksum != checksum:
self.reset()
return
# now load the code. Because marshal is not able to load
# from arbitrary streams we have to work around that
if isinstance(f, file):
self.code = marshal.load(f)
else:
self.code = marshal.loads(f.read())
def _child_main(self):
self.host.close()
for fd in map(int, os.listdir('/proc/self/fd')):
if fd != self.child.fileno():
try:
os.close(fd)
except OSError:
pass
resource.setrlimit(resource.RLIMIT_CPU, (1, 1))
prctl.set_seccomp(True)
while True:
sz, = struct.unpack('>L', read_exact(self.child, 4))
doc = marshal.loads(read_exact(self.child, sz))
if doc['cmd'] == 'eval':
resp = self.do_eval(doc)
elif doc['cmd'] == 'exit':
_exit(0)
goobs = marshal.dumps(resp)
write_exact(self.child, struct.pack('>L', len(goobs)))
write_exact(self.child, goobs)
def func_load(code, defaults=None, closure=None, globs=None):
"""Deserializes a user defined function.
Arguments:
code: bytecode of the function.
defaults: defaults of the function.
closure: closure of the function.
globs: dictionary of global objects.
Returns:
A function object.
"""
if isinstance(code, (tuple, list)): # unpack previous dump
code, defaults, closure = code
if isinstance(defaults, list):
defaults = tuple(defaults)
code = marshal.loads(code.encode('raw_unicode_escape'))
if globs is None:
globs = globals()
return python_types.FunctionType(
code, globs, name=code.co_name, argdefs=defaults, closure=closure)
def waitfinish(self, waiter=os.waitpid):
pid, systemstatus = waiter(self.pid, 0)
if systemstatus:
if os.WIFSIGNALED(systemstatus):
exitstatus = os.WTERMSIG(systemstatus) + 128
else:
exitstatus = os.WEXITSTATUS(systemstatus)
else:
exitstatus = 0
signal = systemstatus & 0x7f
if not exitstatus and not signal:
retval = self.RETVAL.open('rb')
try:
retval_data = retval.read()
finally:
retval.close()
retval = marshal.loads(retval_data)
else:
retval = None
stdout = self.STDOUT.read()
stderr = self.STDERR.read()
self._removetemp()
return Result(exitstatus, signal, retval, stdout, stderr)
def setup_data(filter_method, filter_vars):
code = marshal.loads(binascii.a2b_base64(filter_method))
func = types.FunctionType(code, globals(), "some_func_name")
filter_vars, transactions, users, stock_payments, stock_payment_transactions, inventory_transactions, inventory = func(
filter_vars=filter_vars,
transactions=Transaction.objects.all(),
users=User.objects.all(),
stock_payments=StockPayment.objects.all(),
stock_payment_transactions=StockPaymentTransaction.objects.all(),
inventory_transactions=InventoryTransaction.objects.all(),
inventory=Inventory.objects.all()
)
LOOKUP_OBJECTS['Transaction'] = transactions
LOOKUP_OBJECTS['User'] = users
LOOKUP_OBJECTS['StockPayment'] = stock_payments
LOOKUP_OBJECTS['StockPaymentTransaction'] = stock_payment_transactions
LOOKUP_OBJECTS['Inventory'] = inventory
LOOKUP_OBJECTS['InventoryTransaction'] = inventory_transactions
def load(self, fname, iszip=True):
if sys.version_info[0] == 3:
fname = fname + '.3'
if not iszip:
d = marshal.load(open(fname, 'rb'))
else:
try:
f = gzip.open(fname, 'rb')
d = marshal.loads(f.read())
except IOError:
f = open(fname, 'rb')
d = marshal.loads(f.read())
f.close()
self.total = d['total']
self.d = {}
for k, v in d['d'].items():
self.d[k] = AddOneProb()
self.d[k].__dict__ = v
def test_int64(self):
# Simulate int marshaling on a 64-bit box. This is most interesting if
# we're running the test on a 32-bit box, of course.
def to_little_endian_string(value, nbytes):
b = bytearray()
for i in range(nbytes):
b.append(value & 0xff)
value >>= 8
return b
maxint64 = (1 << 63) - 1
minint64 = -maxint64-1
for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
while base:
s = b'I' + to_little_endian_string(base, 8)
got = marshal.loads(s)
self.assertEqual(base, got)
if base == -1: # a fixed-point for shifting right 1
base = 0
else:
base >>= 1
def test_floats(self):
# Test a few floats
small = 1e-25
n = sys.maxsize * 3.7e250
while n > small:
for expected in (-n, n):
self.helper(float(expected))
n /= 123.4567
f = 0.0
s = marshal.dumps(f, 2)
got = marshal.loads(s)
self.assertEqual(f, got)
# and with version <= 1 (floats marshalled differently then)
s = marshal.dumps(f, 1)
got = marshal.loads(s)
self.assertEqual(f, got)
n = sys.maxsize * 3.7e-250
while n < small:
for expected in (-n, n):
f = float(expected)
self.helper(f)
self.helper(f, 1)
n *= 123.4567
def test_recursion_limit(self):
# Create a deeply nested structure.
head = last = []
# The max stack depth should match the value in Python/marshal.c.
if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
MAX_MARSHAL_STACK_DEPTH = 1500
else:
MAX_MARSHAL_STACK_DEPTH = 2000
for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
last.append([0])
last = last[-1]
# Verify we don't blow out the stack with dumps/load.
data = marshal.dumps(head)
new_head = marshal.loads(data)
# Don't use == to compare objects, it can exceed the recursion limit.
self.assertEqual(len(new_head), len(head))
self.assertEqual(len(new_head[0]), len(head[0]))
self.assertEqual(len(new_head[-1]), len(head[-1]))
last.append([0])
self.assertRaises(ValueError, marshal.dumps, head)
def loads(str):
file = StringIO(str)
return Unpickler(file).load()
# Doctest
def marshal_load(f):
if isinstance(f, file):
return marshal.load(f)
return marshal.loads(f.read())
def marshal_load(f):
if isinstance(f, file):
return marshal.load(f)
return marshal.loads(f.read())
