def load(path, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu)
python类load()的实例源码
def __LaunchTask(self, task, PredCols):
""""""
print('\n---- Begin to deal with %s' % (task))
start = time.time()
if(task == 'MissingValue'):
self.TrainData, self.TestData = self._missing.impute((self.TrainData, self.TestData))
elif(task == 'NewFeature'):
self.TrainData, self.TestData = self._newfeat.create((self.TrainData, self.TestData), PredCols)
elif(task == 'FeatureEncoding'):
with open('%s/featmap.pkl' % self._InputDir, 'rb') as i_file:
d_feat = pickle.load(i_file)
i_file.close()
self.TrainData, self.TestData = self._encoding.ordinal((self.TrainData, self.TestData), d_feat)
elif(task == 'FeatureSelection'):
self.TrainData, self.TestData = self._select.select((self.TrainData, self.TestData))
end = time.time()
print('---- Task %s done, time consumed %ds' % (task, (end - start)))
def load(path, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu)
def load(path, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu)
def load(path, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu)
def load_session(fname=None):
if conf.interactive_shell.lower() == "ipython":
log_interactive.error("There are issues with load_session in ipython. Use python for interactive shell, or use -s parameter to load session")
return
import dill as pickle
if fname is None:
fname = conf.session
try:
s = pickle.load(gzip.open(fname,"rb"))
except IOError:
s = pickle.load(open(fname,"rb"))
scapy_session = builtins.__dict__["scapy_session"]
scapy_session.clear()
scapy_session.update(s)
def saga_cv_cache(*args):
arghash = sha1(repr(args).encode('utf-8')).hexdigest()
fn = "res/baseline_linear_{}.dill".format(arghash)
try:
with open(fn, 'rb') as f:
out = dill.load(f)
logging.info("Loaded cached version.")
except FileNotFoundError:
logging.info("Computing...")
out = saga_cv(*args)
with open(fn, 'wb') as f:
dill.dump(out, f)
return out
def load_session(fname=None):
if conf.interactive_shell.lower() == "ipython":
log_interactive.error("There are issues with load_session in ipython. Use python for interactive shell, or use -s parameter to load session")
return
import dill as pickle
if fname is None:
fname = conf.session
try:
s = pickle.load(gzip.open(fname,"rb"))
except IOError:
s = pickle.load(open(fname,"rb"))
scapy_session = builtins.__dict__["scapy_session"]
scapy_session.clear()
scapy_session.update(s)
def load(path, act_params, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu, act_params=act_params)
def load(path, act_params, num_cpu=16):
"""Load act function that was returned by learn function.
Parameters
----------
path: str
path to the act function pickle
num_cpu: int
number of cpus to use for executing the policy
Returns
-------
act: ActWrapper
function that takes a batch of observations
and returns actions.
"""
return ActWrapper.load(path, num_cpu=num_cpu, act_params=act_params)
def update(self):
if not secure or check_key():
with open(os.path.join(path, 'preql_queries.pickle')) as pq:
self.queries = pickle.load(pq)
with open(os.path.join(path, "synthdb_internal.pickle")) as sdb:
funcs = pickle.load(sdb)
node_topo_funcs.update(funcs['node_topo_funcs'])
link_topo_funcs.update(funcs['link_topo_funcs'])
graph_format.update(funcs['graph_format'])
removers.update(funcs['removers'])
graph_tool_functions.update(funcs['graph_tool_functions'])
graph_generator_functions.update(funcs['graph_generator_functions'])
walkers.update(funcs['walkers'])
topo_formats.update(funcs['topo_formats'])
return json.dumps("Functions updated.")
else:
wrong_key()
def restart_workflow(args, outdir):
with open(args.inputfile, 'r') as infile:
runner = dill.load(infile)
if isinstance(runner.engine, pyccc.Docker) and runner.engine.client is None:
runner.engine.client = runner.engine.connect_to_docker()
engine, RunnerClass = get_execution_env(args)
assert RunnerClass is runner.__class__
if args.setoutput:
set_ui_outputs(runner, args)
print ' ---- RESTARTING WORKFLOW "%s" ----\n' % runner.workflow.name
run_workflow(runner, outdir)
if args.dumptasks:
dump_all_tasks(runner, outdir)
def process_input_file(inputfile):
""" Figure out whether we're being passed a file, a description of a file, or just raw JSON
"""
try:
jsraw = _get_json(inputfile)
except ValueError:
pass
else:
print jsraw
inputjson = json.loads(jsraw)
return inputjson
ext = inputfile.split('.')[-1]
if ext in ('js', 'json', 'yml', 'yaml'):
with open(inputfile, 'r') as infile:
inputjson = yaml.load(infile)
else:
with open(inputfile, 'r') as infile:
inputjson = {'filename': inputfile,
'content': infile.read()}
return inputjson
def test_create(self, mock):
value = 1
function_name = 'test_function'
@Lambda(name=function_name, bucket='test', key='test', client=self.client)
def foo():
return value
package = DeploymentPackage(foo)
zfp = zipfile.ZipFile(StringIO(package.zip_bytes(foo.dumped_code)), "r")
func = dill.load(zfp.open('.lambda.dump'))
self.assertEqual(func(), value)
resp_create = foo.create()
self.assertEqual(resp_create['FunctionName'], function_name)
# moto doesn't support ZipFile only lambda deployments, while
# aws doen't allow other arguments when scpesifying ZipFile argument
#resp_get = foo.get()
#self.assertEqual(resp_get['Configuration']['FunctionName'], function_name)
def run_sample_type_prediction(tag_to_val, mapped_terms, real_props):
# Load the dilled vectorizer and model
vectorizer_f = pr.resource_filename(__name__, join("predict_sample_type", "sample_type_vectorizor.dill"))
classifier_f = pr.resource_filename(__name__, join("predict_sample_type", "sample_type_classifier.dill"))
with open(vectorizer_f, "rb") as f:
vectorizer = dill.load(f)
with open(classifier_f, "rb") as f:
model = dill.load(f)
# Make sample-type prediction
feat_v = vectorizer.convert_to_features(
get_ngrams_from_tag_to_val(tag_to_val),
mapped_terms)
predicted, confidence = model.predict(
feat_v,
mapped_terms,
real_props)
return predicted, confidence
def dill_words(num_words, fname="words.dill"):
fname = os.path.join(os.path.dirname(os.path.realpath(__file__)), fname)
try:
if os.path.isfile(fname):
words = dill.load(open(fname, "rb"))
if(len(words) < ip_handling.iutils.get_ipv6_word_possibilities()):
os.remove(fname)
raise Exception # go into except block to reload words
return words
else:
words = load_words(num_words)
if(len(words) < ip_handling.iutils.get_ipv6_word_possibilities()):
raise Exception # go into except block to reload words
dill.dump(words, open(fname, "wb"))
return words
except:
try:
words = load_words(num_words)
if(len(words) < ip_handling.iutils.get_ipv6_word_possibilities()):
raise Exception # go into except block to reload words
dill.dump(words, open(fname, "wb"))
return words
except:
return load_words(num_words)
def _load_file(path):
"""
Use to load a file with dill, used for loading puzzle. file should have \
been wrote with dill.
:param str path: path to the file to load
:return: object (excepted [Puzzle](doc/puzzle.md) but no check is made)
"""
try:
with open(path, "rb") as e:
puzzle.Puzzle.dynamique_type()
return dill.load(e)
except Exception as e:
print "Exception while loading file %s" % path
print e
return None
def get_files(folder_name, algorithm, args):
foldersTmp = os.listdir(folder_name)
folders = []
for folder in foldersTmp:
if folder[0] == '.':
continue
folders.append(folder)
imgs = []
for folder in folders:
path = folder_name + folder + '/'
if not os.path.isdir(path):
continue
files = os.listdir(path)
for file_str in files:
complete_file_str = str((os.path.join(path, file_str)))
if os.path.isfile(complete_file_str) and (complete_file_str.endswith('.jpg') or complete_file_str.endswith('.JPG')):
imgs.append((os.path.join(path, file_str), folder))
return imgs
# load instances from filename
def get_files(folder_name, algorithm, args):
foldersTmp = os.listdir(folder_name)
folders = []
for folder in foldersTmp:
if folder[0] == '.':
continue
folders.append(folder)
imgs = []
for folder in folders:
path = folder_name + folder + '/'
if not os.path.isdir(path):
continue
files = os.listdir(path)
for file_str in files:
complete_file_str = str((os.path.join(path, file_str)))
if os.path.isfile(complete_file_str) and (complete_file_str.endswith('.jpg') or complete_file_str.endswith('.JPG')):
imgs.append((os.path.join(path, file_str), folder))
return imgs
# load instances from filename
def main(prepare, use, do, get, params, debug):
if get is not None:
do = get
if prepare is not None and use in ['ht', 'yjb', 'yh', 'gf', 'xq']:
user = easytrader.use(use, debug)
user.prepare(prepare)
with open(ACCOUNT_OBJECT_FILE, 'wb') as f:
dill.dump(user, f)
if do is not None:
with open(ACCOUNT_OBJECT_FILE, 'rb') as f:
user = dill.load(f)
if len(params) > 0:
result = getattr(user, do)(*params)
else:
result = getattr(user, do)
json_result = json.dumps(result, indent=4, ensure_ascii=False, sort_keys=True)
click.echo(json_result)
def read(self):
"""
Reads from persisted files.
Returns
-------
sparse matrix
The train data
sparse matrix
The label data
ThesaurusReader
Unpickled ThesaurusReader object
"""
print('Reading persisted features')
X = self._load_sparse_csr(self._persist_name('X'))
y = self._load_sparse_csr(self._persist_name('y'))
with open(self._persist_name('TR'), mode='rb') as f:
tr = pickle.load(f)
return X, y, tr
def create_processed_dataframe_from_mongo(dbname):
'''
INPUT
- dbname: this is the name of the mongo database where the
information will be extracted from
OUTPUT
- df
Returns a dataframe that has everything needed in order to do modelling
'''
df = extract_user_information_from_mongo(dbname, 'topictweets')
# df = pd.read_csv('data/clinton_df.csv')
# df.id = df.id.apply(str)
feature_dict = extract_feature_information_from_mongo(dbname,
'timelinetweets')
# with open('data/clinton_tweets_dict.pkl', 'r') as f:
# feature_dict = pickle.load(f)
df = df.drop_duplicates(subset='id', keep='last')
users_who_tweeted = set(feature_dict.keys())
dfusers_who_tweeted = df[df.id.isin(users_who_tweeted)]
# subset the initial user dataframe to have ONLY the users who tweeted
df = combine_user_info_with_feature_dict(dfusers_who_tweeted, feature_dict)
df = process_feature_information_for_modelling(df, feature_dict)
df = drop_unnecessary_columns_from_test_data(df)
return df
def restore_state(self, path):
"""Returns loaded state"""
try:
with open(path, 'rb') as f:
if self.encrypt:
state = pickle.loads(self.decrypt_data(pickle.load(f)))
else:
state = pickle.load(f)
LOG.debug("Restoring state successs")
except Exception as e:
LOG.debug("Restoring state from %s failed with %s" % (
path, e))
state = StateMachine(self.bot, state_path=path)
LOG.debug("Successfully inicialized new state.")
return state
def load_session(fname=None):
if conf.interactive_shell.lower() == "ipython":
log_interactive.error("There are issues with load_session in ipython. Use python for interactive shell, or use -s parameter to load session")
return
import dill as pickle
if fname is None:
fname = conf.session
try:
s = pickle.load(gzip.open(fname,"rb"))
except IOError:
s = pickle.load(open(fname,"rb"))
scapy_session = builtins.__dict__["scapy_session"]
scapy_session.clear()
scapy_session.update(s)
def restore(file_name="dump.bin"):
return pickle.load(open(file_name, 'rb'))
# class Encoding:
# pass
# @extension
# class Math:
# WOOOT? just
# import math as Math
# def __getattr__(self, attr):
# import sys
# import math
# # ruby method_missing !!!
# import inspect
# for name, obj in inspect.getmembers(sys.modules['math']):
# if name==attr: return obj
# return False
def load(path, num_cpu=16):
with open(path, "rb") as f:
model_data, act_params = dill.load(f)
act = deepq.build_act(**act_params)
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
with tempfile.TemporaryDirectory() as td:
arc_path = os.path.join(td, "packed.zip")
with open(arc_path, "wb") as f:
f.write(model_data)
zipfile.ZipFile(arc_path, 'r', zipfile.ZIP_DEFLATED).extractall(td)
U.load_state(os.path.join(td, "model"))
return ActWrapper(act, act_params)
def restore(self):
print('Try load previous plot....')
try:
self._since_beginning = dill.load(open(self.LOGDIR+'log.pkl', "r"))
self._iter = dill.load(open(self.LOGDIR+'iteration.pkl', "r"))
# self._iter += 1
print('Restore plot from iter: '+str(self._iter))
return self._iter
except Exception, e:
print('Previous plot unfounded')
return 0
print('')
def __MergeData(cls, InputDir, OutputDir, mode):
""""""
if(mode == 'train'):
ActionDataFile = '%s/train_2016_v2.csv' % InputDir
OutputFile = '%s/train.pkl' % OutputDir
else:
ActionDataFile = '%s/sample_submission.csv' % InputDir
OutputFile = '%s/test.pkl' % OutputDir
print(OutputFile)
PropertyDataFile = '%s/properties_2016.csv' % InputDir
## load
ActionData = pd.read_csv(ActionDataFile, parse_dates=['transactiondate'])
PropertyData = pd.read_csv(PropertyDataFile)
## left join
MergedData = ActionData.merge(PropertyData, how='left', on='parcelid')
## output into pkl file
if (os.path.exists(OutputDir) == False):
os.makedirs(OutputDir)
with open(OutputFile, 'wb') as o_file:
pickle.dump(MergedData, o_file, -1)
o_file.close()
return
## split rawcensustractandblock into census, tract and block
def __ParseCTB(cls, InputDir, OutputDir, mode):
""""""
if(mode == 'train'):
InputFile = '%s/train.pkl' % InputDir
OutputFile = '%s/train.pkl' % OutputDir
else:
InputFile = '%s/test.pkl' % InputDir
OutputFile = '%s/test.pkl' % OutputDir
## load
with open(InputFile, 'rb') as i_file:
df_data = pickle.load(i_file)
i_file.close()
## extract census, tract and block identifies
df_data['rawcensustractandblock'] = (df_data['rawcensustractandblock'] * 1000).astype(np.float64).astype(np.int64)
df_data['fipsid'] = ((df_data['rawcensustractandblock'] / 10000000).astype(np.int64)).astype(str)
df_data['tractandblock'] = df_data['rawcensustractandblock'] % 10000000
df_data['tractid'] = ((df_data['tractandblock'] / 10).astype(np.int64)).astype(str)
df_data['blockid'] = ((df_data['tractandblock'] % 10).astype(np.int64)).astype(str)
df_data.drop(['fips', 'rawcensustractandblock', 'tractandblock'], axis = 1, inplace = True)
## output into pkl file
if (os.path.exists(OutputDir) == False):
os.makedirs(OutputDir)
with open(OutputFile, 'wb') as o_file:
pickle.dump(df_data, o_file, -1)
o_file.close()
return
def __SplitData(cls, InputDir, OutputDir, mode):
""""""
if(mode == 'train'):
InputFileData = '%s/train.pkl' % InputDir
else:
InputFileData = '%s/test.pkl' % InputDir
InputFileFeatMap = '%s/featmap.pkl' % InputDir
## load
with open(InputFileData, 'rb') as i_file:
df_data = pickle.load(i_file)
i_file.close()
with open(InputFileFeatMap, 'rb') as i_file:
d_feat = pickle.load(i_file)
i_file.close()
if (os.path.exists(OutputDir) == False):
os.makedirs(OutputDir)
with open('%s/featmap.pkl' % OutputDir, 'wb') as o_file:
pickle.dump(d_feat, o_file, -1)
o_file.close()
## output into individual pkl files
for i in range(12):
month = i + 1
df_MonthData = df_data[(df_data['transactiondate'].dt.month == month)]
with open('%s/%s.pkl'% (OutputDir, month), 'wb') as o_file:
pickle.dump(df_MonthData, o_file, -1)
o_file.close()
return
## launch single task
