def save_merge(self):
if not self.cachefile:
return
glf = bb.utils.lockfile(self.cachefile + ".lock")
data = self.cachedata
for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
f = os.path.join(os.path.dirname(self.cachefile), f)
try:
with open(f, "rb") as fd:
p = pickle.Unpickler(fd)
extradata, version = p.load()
except (IOError, EOFError):
os.unlink(f)
continue
if version != self.__class__.CACHE_VERSION:
os.unlink(f)
continue
self.merge_data(extradata, data)
os.unlink(f)
with open(self.cachefile, "wb") as f:
p = pickle.Pickler(f, -1)
p.dump([data, self.__class__.CACHE_VERSION])
bb.utils.unlockfile(glf)
python类Unpickler()的实例源码
def __init__(self, f, objmap, shortcuts=None):
pickle.Unpickler.__init__(self, f)
if shortcuts:
objmap = dict((k % shortcuts, v % shortcuts)
for k, v in objmap.items())
self._objmap = objmap
def test_bad_init(self):
# Test issue3664 (pickle can segfault from a badly initialized Pickler).
# Override initialization without calling __init__() of the superclass.
class BadPickler(pickle.Pickler):
def __init__(self): pass
class BadUnpickler(pickle.Unpickler):
def __init__(self): pass
self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
def test_priming_unpickler_memo(self):
# Verify that we can set the Unpickler's memo attribute.
data = ["abcdefg", "abcdefg", 44]
f = io.BytesIO()
pickler = self.pickler_class(f)
pickler.dump(data)
first_pickled = f.getvalue()
f = io.BytesIO()
primed = self.pickler_class(f)
primed.memo = pickler.memo
primed.dump(data)
primed_pickled = f.getvalue()
unpickler = self.unpickler_class(io.BytesIO(first_pickled))
unpickled_data1 = unpickler.load()
self.assertEqual(unpickled_data1, data)
primed = self.unpickler_class(io.BytesIO(primed_pickled))
primed.memo = unpickler.memo
unpickled_data2 = primed.load()
primed.memo.clear()
self.assertEqual(unpickled_data2, data)
self.assertTrue(unpickled_data2 is unpickled_data1)
def __getitem__(self, key):
try:
value = self.cache[key]
except KeyError:
f = BytesIO(self.dict[key.encode(self.keyencoding)])
value = Unpickler(f).load()
if self.writeback:
self.cache[key] = value
return value
def set_location(self, key):
(key, value) = self.dict.set_location(key)
f = BytesIO(value)
return (key.decode(self.keyencoding), Unpickler(f).load())
def previous(self):
(key, value) = self.dict.previous()
f = BytesIO(value)
return (key.decode(self.keyencoding), Unpickler(f).load())
def first(self):
(key, value) = self.dict.first()
f = BytesIO(value)
return (key.decode(self.keyencoding), Unpickler(f).load())
def last(self):
(key, value) = self.dict.last()
f = BytesIO(value)
return (key.decode(self.keyencoding), Unpickler(f).load())
def find_class(self, module, name):
""" This override is here to help pickle find the modules that classes are defined in.
It does three things:
1) remaps the "PackagedFunction" class from pyccc to the `source.py` module.
2) Remaps any classes created in the client's '__main__' to the `source.py` module
3) Creates on-the-fly modules to store any other classes present in source.py
References:
This is a modified version of the 2-only recipe from
https://wiki.python.org/moin/UsingPickle/RenamingModules.
It's been modified for 2/3 cross-compatibility """
import pickle
modname = self.RENAMETABLE.get(module, module)
try:
# can't use ``super`` here (not 2/3 compatible)
klass = pickle.Unpickler.find_class(self, modname, name)
except (ImportError, RuntimeError):
definition = getattr(source, name)
newmod = _makemod(modname)
sys.modules[modname] = newmod
setattr(newmod, name, definition)
klass = pickle.Unpickler.find_class(self, newmod.__name__, name)
klass.__module__ = module
return klass
def load(fp):
return pickle.Unpickler(fp).load()
def start():
if os.path.exists(fileName):
f = open(fileName, "rb")
d = pickle.Unpickler(f)
data = d.load()
f.close()
else:
data = {}
return data
#####################################################################################################################################################
def start():
if os.path.exists(fileName):
f = open(fileName, "rb")
d = pickle.Unpickler(f)
data = d.load()
f.close()
else:
data = {}
return data
########################################################################################################################
def __init__(self, filename, flag='c', protocol=None, keyencoding='utf-8'):
self.db = filename
self.flag = flag
self.dict = {}
with dbm.open(self.db, self.flag) as db:
for k in db.keys():
v = BytesIO(db[k])
self.dict[k] = Unpickler(v).load()
shelve.Shelf.__init__(self, self.dict, protocol, False, keyencoding)
def __init__(self):
# ustawienia przegl?darki
binary = FirefoxBinary('/home/endo93/Firefox 46.0/firefox')
self.firefox = webdriver.Firefox(firefox_binary=binary)
os.system('rm sql/filterservice.log')
self.Wczytaj_Adresy_URL()
self.Zapisz_Adresy_URL()
# pickler - zapis/odczyt produktów z pliku
pickler = None
plik = None
if os.path.exists('sql/filterservice.prod'):
Kierownik.odczyt_produktow = True
plik = open('sql/filterservice.prod', 'rb')
pickler = pickle.Unpickler(plik)
else: # do zapisywania
plik = open('sql/filterservice.prod', 'wb')
pickler = pickle.Pickler(plik, pickle.HIGHEST_PROTOCOL)
# dla ka?dego adresu url
for url in self.adresy_url:
self.Dodaj_Produkt_Do_Bazy_Danych(url, 'Inne', 'Inne',
'Inne', pickler)
plik.close()
self.firefox.close()
def __init__(self):
# ustawienia przegl?darki
binary = FirefoxBinary('/home/endo93/Firefox 46.0/firefox')
self.firefox = webdriver.Firefox(firefox_binary=binary)
os.system('rm sql/adler.log')
self.Wczytaj_Adresy_URL()
self.Zapisz_Adresy_URL()
# pickler - zapis/odczyt produktów z pliku
pickler = None
plik = None
if os.path.exists('sql/adler.prod'):
Kierownik.odczyt_produktow = True
plik = open('sql/adler.prod', 'rb')
pickler = pickle.Unpickler(plik)
else: # do zapisywania
plik = open('sql/adler.prod', 'wb')
pickler = pickle.Pickler(plik, pickle.HIGHEST_PROTOCOL)
# dla ka?dego adresu url
for typ in self.adresy_url:
for dziedzina in self.adresy_url[typ]:
for rodzaj in self.adresy_url[typ][dziedzina]:
for url in self.adresy_url[typ][dziedzina][rodzaj]:
self.Dodaj_Produkt_Do_Bazy_Danych(url, typ, dziedzina,
rodzaj, pickler)
plik.close()
self.firefox.close()
def __init__(self):
# ustawienia przegl?darki
binary = FirefoxBinary('/home/endo93/Firefox 46.0/firefox')
self.firefox = webdriver.Firefox(firefox_binary=binary)
os.system('rm sql/polstar.log')
self.Wczytaj_Adresy_URL()
self.Zapisz_Adresy_URL()
# pickler - zapis/odczyt produktów z pliku
pickler = None
plik = None
if os.path.exists('sql/polstar.prod'):
Kierownik.odczyt_produktow = True
plik = open('sql/polstar.prod', 'rb')
pickler = pickle.Unpickler(plik)
else: # do zapisywania
plik = open('sql/polstar.prod', 'wb')
pickler = pickle.Pickler(plik, pickle.HIGHEST_PROTOCOL)
# dla ka?dego adresu url
for typ in self.adresy_url:
for dziedzina in self.adresy_url[typ]:
for url in self.adresy_url[typ][dziedzina]:
self.Dodaj_Produkt_Do_Bazy_Danych(url, typ, dziedzina,
'Inne', pickler)
plik.close()
self.firefox.close()
def __init__(self, f, objmap, shortcuts=None):
pickle.Unpickler.__init__(self, f)
if shortcuts:
objmap = dict((k % shortcuts, v % shortcuts)
for k, v in objmap.items())
self._objmap = objmap
def loads(self, buf):
f = StringIO(buf)
u = pickle.Unpickler(f)
return u.load()
def loads(self, buf):
f = StringIO(buf)
u = pickle.Unpickler(f)
return u.load()
