utils.py 文件源码

def _restore_dict(self, path, read_only, cache_size):
    # ====== already exist ====== #
    if os.path.exists(path):
      if os.path.getsize(path) == 0:
        if read_only:
          raise Exception('File at path:"%s" has zero size, no data '
                          'found in (read-only mode).' % path)
      file = open(str(path), mode='rb+')
      if file.read(len(MmapDict.HEADER)) != MmapDict.HEADER:
        raise Exception('Given file is not in the right format '
                        'for MmapDict.')
      # 48 bytes for the file size
      max_position = int(file.read(MmapDict.SIZE_BYTES))
      # length of pickled indices dictionary
      dict_size = int(file.read(MmapDict.SIZE_BYTES))
      # read dictionary
      file.seek(max_position)
      pickled_indices = file.read(dict_size)
      self._indices_dict = async(lambda: cPickle.loads(pickled_indices))()
    # ====== create new file from scratch ====== #
    else:
      file = open(str(path), mode='wb+')
      file.write(MmapDict.HEADER)
      # just write the header
      header = ('%' + str(MmapDict.SIZE_BYTES) + 'd') % \
          (len(MmapDict.HEADER) + MmapDict.SIZE_BYTES * 2)
      file.write(header.encode())
      # write the length of Pickled indices dictionary
      data_size = ('%' + str(MmapDict.SIZE_BYTES) + 'd') % 0
      file.write(data_size.encode())
      file.flush()
      # init indices dict
      self._indices_dict = {}
    # ====== create Mmap from offset file ====== #
    self._file = file
    self._mmap = mmap.mmap(file.fileno(), length=0, offset=0,
                           flags=mmap.MAP_SHARED)
    self._increased_indices_size = 0. # in MB
    # store all the (key, value) recently added
    self._cache_dict = {}