python类HIGHEST_PROTOCOL的实例源码

def save_queue(EXPERIENCE_MEMORY):
    with open('saved nets/saved_queue_new.pkl','wb') as f:
        cPickle.dump(EXPERIENCE_MEMORY,f,protocol=cPickle.HIGHEST_PROTOCOL)

    call(['rm','saved nets/saved_queue.pkl'])
    call(['mv','saved nets/saved_queue_new.pkl','saved nets/saved_queue.pkl'])

def save_queue(EXPERIENCE_MEMORY):
    with open('saved_DDQN/double_dqn_queue_new.pkl','wb') as f:
        cPickle.dump(EXPERIENCE_MEMORY,f,protocol=cPickle.HIGHEST_PROTOCOL)

    call(['rm','saved_DDQN/double_dqn_queue.pkl'])
    call(['mv','saved_DDQN/double_dqn_queue_new.pkl','saved_DDQN/double_dqn_queue.pkl'])

def save_params(self, fname):
        layers = [self.S] + lasagne.layers.get_all_layers(self.A)
        params = chain.from_iterable(l.get_params() for l in layers)
        params = lasagne.utils.unique(params)

        npy_list = [param.get_value(borrow=True) for param in params]

        with open(fname + ".pkl", 'wb') as f:
            pickle.dump(npy_list, f, pickle.HIGHEST_PROTOCOL)

def save(self, fname):
    """Save a pickled version of the embedding into `fname`."""

    vec = self.vectors
    voc = self.vocabulary.getstate()
    state = (voc, vec)
    with open(fname, 'wb') as f:
      pickle.dump(state, f, protocol=pickle.HIGHEST_PROTOCOL)

def topickle(self, filename):
        # remove unpicklable attributes
        warnfunc = self._warnfunc
        self.set_warnfunc(None)
        values = self.config.values
        del self.config.values
        domains = self.domains
        del self.domains
        picklefile = open(filename, 'wb')
        # remove potentially pickling-problematic values from config
        for key, val in list(vars(self.config).items()):
            if key.startswith('_') or \
               isinstance(val, types.ModuleType) or \
               isinstance(val, types.FunctionType) or \
               isinstance(val, class_types):
                del self.config[key]
        try:
            pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)
        finally:
            picklefile.close()
        # reset attributes
        self.domains = domains
        self.config.values = values
        self.set_warnfunc(warnfunc)

    # --------- ENVIRONMENT INITIALIZATION -------------------------------------

def save(self,pickle_name):
        print ('saving data to ',pikckle_name)
        if self.train :
            f = open(pickle_name, 'wb')
            cPickle.dump(self.qValues, f, protocol=cPickle.HIGHEST_PROTOCOL)
            f.close()

def save(self,pickle_name):
        if self.train :
            print ('saving data to ',pickle_name)
            f = open(pickle_name, 'wb')
            cPickle.dump(self.weights, f, protocol=cPickle.HIGHEST_PROTOCOL)
            f.close()

def save_object(obj, path):
    """saves an object to a file"""
    with open(path, 'wb') as output:
        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)

def file_pickle(pickle_files, save, force):
  if force or not os.path.exists(pickle_files):
    try:
      with open(pickle_files,'wb') as f:
        pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
    except Exception as e:
        print('Unable to save data to', pickle_files, ':', e)
    return pickle_files

def __getstate__(self):
    if not self._new_args_called:
      raise RuntimeError(
          "You must use argument `protocol=cPickle.HIGHEST_PROTOCOL` "
          "when using `pickle` or `cPickle` to be able pickling NoSQL.")
    self._new_args_called = False
    return self.path, self.read_only, self.cache_size

def __getstate__(self):
    if not self._new_args_called:
      raise RuntimeError(
          "You must use argument `protocol=cPickle.HIGHEST_PROTOCOL` "
          "when using `pickle` or `cPickle` to be able pickling Dataset.")
    self._new_args_called = False
    return self.path, self.read_only

def add_recipes(self, recipes, name, override=False):
    """
    Parameters
    ----------
    """
    # ====== validate arguments ====== #
    if not is_string(name):
      raise ValueError("`name` must be string, but given: %s" % str(type(name)))
    if name in self._saved_recipes and not override:
      raise ValueError("Cannot override pre-defined RECIPE with name: '%s'"
                      % name)
    # ====== validate recipes list ====== #
    if isinstance(recipes, RecipeList):
      recipes = tuple(recipes._recipes)
    else:
      tmp = []
      for rcp in as_tuple(recipes, t=FeederRecipe):
        if isinstance(rcp, RecipeList):
          tmp += list(rcp._recipes)
        else:
          tmp.append(rcp)
      recipes = tuple(tmp)
    # ====== store the recipes to disk ====== #
    path = os.path.join(self.recipe_path, name)
    with open(path, 'wb') as f:
      cPickle.dump(recipes, f, protocol=cPickle.HIGHEST_PROTOCOL)
    # ====== update local recipes list ====== #
    self._saved_recipes[name] = recipes
    return self

def flush(self):
    for dtype, shape, data, path in self._data_map.values():
      if hasattr(data, 'flush'):
        data.flush()
      elif data is not None: # Flush pickling data
        with open(path, 'wb') as f:
          cPickle.dump(data, f, protocol=cPickle.HIGHEST_PROTOCOL)

def func_to_str(func):
  # conver to byte
  code = cPickle.dumps(array("B", marshal.dumps(func.__code__)),
                       protocol=cPickle.HIGHEST_PROTOCOL)
  closure = None
  if func.__closure__ is not None:
    print("[WARNING] function: %s contains closure, which cannot be "
          "serialized." % str(func))
    closure = tuple([c.cell_contents for c in func.__closure__])
  defaults = func.__defaults__
  return (code, closure, defaults)

def __init__(self, func, *args, **kwargs):
    super(functionable, self).__init__()
    self._function = func
    self.__name__ = self._function.__name__
    try: # sometime cannot get the source
      self._source = inspect.getsource(self._function)
    except Exception as e:
      print("[WARNING] Cannot get source code of function:", func,
            "(error:%s)" % str(e))
      self._source = None
    # try to pickle the function directly
    try:
      self._sandbox = cPickle.dumps(self._function,
          protocol=cPickle.HIGHEST_PROTOCOL)
    except Exception:
      self._sandbox = _serialize_function_sandbox(func, self._source)
    # ====== store argsmap ====== #
    argspec = inspect.getargspec(func)
    argsmap = OrderedDict([(i, _ArgPlaceHolder_()) for i in argspec.args])
    # store defaults
    if argspec.defaults is not None:
      for name, arg in zip(argspec.args[::-1], argspec.defaults[::-1]):
        argsmap[name] = arg
    # update positional arguments
    for name, arg in zip(argspec.args, args):
      argsmap[name] = arg
    # update kw arguments
    argsmap.update(kwargs)
    self._argsmap = argsmap

  # ==================== Pickling methods ==================== #

def is_pickleable(x):
  try:
    cPickle.dumps(x, protocol=cPickle.HIGHEST_PROTOCOL)
    return True
  except cPickle.PickleError:
    return False

def __getstate__(self):
    if not self._new_args_called:
      raise RuntimeError(
          "You must use argument `protocol=cPickle.HIGHEST_PROTOCOL` "
          "when using `pickle` or `cPickle` to be able pickling NNOp.")
    self._new_args_called = False
    # add nnops here so all related NNOps are saved
    return self._save_states, self.nnops

def read_dataset(data_dir):
    pickle_filename = "celebA.pickle"
    pickle_filepath = os.path.join(data_dir, pickle_filename)
    if not os.path.exists(pickle_filepath):
        # utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
        celebA_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
        dir_path = os.path.join(data_dir, celebA_folder)
        if not os.path.exists(dir_path):
            print ("CelebA dataset needs to be downloaded and unzipped manually")
            print ("Download from: %s" % DATA_URL)
            raise ValueError("Dataset not found")

        result = create_image_lists(dir_path)
        print ("Training set: %d" % len(result['train']))
        print ("Test set: %d" % len(result['test']))
        print ("Validation set: %d" % len(result['validation']))
        print ("Pickling ...")
        with open(pickle_filepath, 'wb') as f:
            pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
    else:
        print ("Found pickle file!")

    with open(pickle_filepath, 'rb') as f:
        result = pickle.load(f)
        celebA = CelebA_Dataset(result)
        del result
    return celebA

def main(params):

  imgs = json.load(open(params['input_json'], 'r'))
  itow = json.load(open(params['dict_json'], 'r'))['ix_to_word']
  wtoi = {w:i for i,w in itow.items()}

  imgs = imgs['images']

  ngram_words, ngram_idxs, ref_len = build_dict(imgs, wtoi, params)

  cPickle.dump({'document_frequency': ngram_words, 'ref_len': ref_len}, open(params['output_pkl']+'-words.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)
  cPickle.dump({'document_frequency': ngram_idxs, 'ref_len': ref_len}, open(params['output_pkl']+'-idxs.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)

def read_dataset(data_dir):
    pickle_filename = "flowers_data.pickle"
    pickle_filepath = os.path.join(data_dir, pickle_filename)
    if not os.path.exists(pickle_filepath):
        utils.maybe_download_and_extract(data_dir, DATA_URL, is_tarfile=True)
        flower_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
        result = create_image_lists(os.path.join(data_dir, flower_folder))
        print "Training set: %d" % len(result['train'])
        print "Test set: %d" % len(result['test'])
        print "Validation set: %d" % len(result['validation'])
        print "Pickling ..."
        with open(pickle_filepath, 'wb') as f:
            pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
    else:
        print "Found pickle file!"

    with open(pickle_filepath, 'rb') as f:
        result = pickle.load(f)
        training_images = result['train']
        testing_images = result['test']
        validation_images = result['validation']

        del result

    print ("Training: %d, Validation: %d, Test: %d" % (
        len(training_images), len(validation_images), len(testing_images)))
    return training_images, testing_images, validation_images