def test_signals(self):
global received_pre_delete
global received_post_save
received_pre_delete = False
received_post_save = False
def handle_pre_delete(**kwargs):
global received_pre_delete
received_pre_delete = True
signals.pre_delete.connect(handle_pre_delete, sender=TestC)
def handle_post_save(**kwargs):
global received_post_save
received_post_save = True
signals.post_save.connect(handle_post_save, sender=TestC)
a = TestC()
a.put()
a.delete()
self.assertTrue(received_pre_delete)
self.assertTrue(received_post_save)
python类put()的实例源码
def test_batch_signals(self):
global received_pre_delete
global received_post_save
received_pre_delete = False
received_post_save = False
def handle_pre_delete(**kwargs):
global received_pre_delete
received_pre_delete = True
signals.pre_delete.connect(handle_pre_delete, sender=TestC)
def handle_post_save(**kwargs):
global received_post_save
received_post_save = True
signals.post_save.connect(handle_post_save, sender=TestC)
a = TestC()
db.put([a])
db.delete([a])
self.assertTrue(received_pre_delete)
self.assertTrue(received_post_save)
# Test serialization
def test_cleanup(self):
signals.pre_delete.connect(cleanup_relations, sender=TestC)
c1 = TestC()
c2 = TestC()
db.put((c1, c2))
TestModelRel(modelrel=c1).put()
child = SigChild(owner=c1, rel=c2)
child.put()
self.assertEqual(TestC.all().count(), 2)
self.assertEqual(SigChild.all().count(), 1)
self.assertEqual(TestModelRel.all().count(), 1)
c1.delete()
signals.pre_delete.disconnect(cleanup_relations, sender=TestC)
self.assertEqual(SigChild.all().count(), 0)
self.assertEqual(TestC.all().count(), 0)
self.assertEqual(TestModelRel.all().count(), 0)
def update(cls, cursor=None):
logging.info('Upgrading schema for Crash Reports (Cursor = %s)' % unicode(cursor))
query = CrashReport.all()
if cursor:
query.with_cursor(cursor)
crash_reports = list()
for crash_report in query.fetch(limit=BATCH_SIZE):
crash_report.version = '2'
crash_report.state = 'unresolved'
crash_reports.append(crash_report)
if crash_reports:
updated = len(crash_reports)
logging.info('Updating %s entities', updated)
# update
db.put(crash_reports)
Search.add_crash_reports(crash_reports)
# schedule next request
deferred.defer(SchemaUpdater.update, cursor=query.cursor())
def _perform_backup_complete(
operation, job_id, kind, backup_info_pk, gcs_path_prefix, filenames, queue):
backup_info = BackupInformation.get(backup_info_pk)
if backup_info:
if job_id in backup_info.active_jobs:
backup_info.active_jobs.remove(job_id)
backup_info.completed_jobs = list(
set(backup_info.completed_jobs + [job_id]))
filenames = [GCSUtil.add_gs_prefix_if_missing(name) for name in filenames]
kind_backup_files = backup_info.get_kind_backup_files([kind])[0]
if kind_backup_files:
kind_backup_files.files = list(set(kind_backup_files.files + filenames))
else:
kind_backup_files = backup_info.create_kind_backup_files(kind, filenames)
db.put((backup_info, kind_backup_files), force_writes=True)
if operation.status == utils.DatastoreAdminOperation.STATUS_COMPLETED:
deferred.defer(finalize_backup_info, backup_info.key(),
gcs_path_prefix,
_url=config.DEFERRED_PATH,
_queue=queue,
_transactional=True)
else:
logging.warn('BackupInfo was not found for %s', backup_info_pk)
def _save_states(self, state, serialized_readers_entity):
"""Run transaction to save state.
Args:
state: a model.MapreduceState entity.
serialized_readers_entity: a model._HugeTaskPayload entity containing
json serialized input readers.
Returns:
False if a fatal error is encountered and this task should be dropped
immediately. True if transaction is successful. None if a previous
attempt of this same transaction has already succeeded.
"""
mr_id = state.key().id_or_name()
fresh_state = model.MapreduceState.get_by_job_id(mr_id)
if not self._check_mr_state(fresh_state, mr_id):
return False
if fresh_state.active_shards != 0:
logging.warning(
"Mapreduce %s already has active shards. Looks like spurious task "
"execution.", mr_id)
return None
config = util.create_datastore_write_config(state.mapreduce_spec)
db.put([state, serialized_readers_entity], config=config)
return True
def _create_and_save_state(cls, mapreduce_spec, _app):
"""Save mapreduce state to datastore.
Save state to datastore so that UI can see it immediately.
Args:
mapreduce_spec: model.MapreduceSpec,
_app: app id if specified. None otherwise.
Returns:
The saved Mapreduce state.
"""
state = model.MapreduceState.create_new(mapreduce_spec.mapreduce_id)
state.mapreduce_spec = mapreduce_spec
state.active = True
state.active_shards = 0
if _app:
state.app_id = _app
config = util.create_datastore_write_config(mapreduce_spec)
state.put(config=config)
return state
def update(self,tablename,query,update_fields):
# self.db['_lastsql'] = self._update(tablename,query,update_fields)
(items, tablename, fields) = self.select_raw(query)
counter = 0
for item in items:
for field, value in update_fields:
setattr(item, field.name, self.represent(value,field.type))
item.put()
counter += 1
LOGGER.info(str(counter))
return counter
def insert(self,table,fields):
dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
# table._db['_lastsql'] = self._insert(table,fields)
tmp = table._tableobj(**dfields)
tmp.put()
rid = Reference(tmp.key().id())
(rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
return rid
def bulk_insert(self,table,items):
parsed_items = []
for item in items:
dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
parsed_items.append(table._tableobj(**dfields))
gae.put(parsed_items)
return True
def test_serializer(self, format='json'):
from django.core import serializers
created = datetime.now()
x = SerializeModel(key_name='blue_key', name='blue', count=4)
x.put()
SerializeModel(name='green', count=1, created=created).put()
data = serializers.serialize(format, SerializeModel.all())
db.delete(SerializeModel.all().fetch(100))
for obj in serializers.deserialize(format, data):
obj.save()
self.validate_state(
('key.name', 'name', 'count', 'created'),
(None, 'green', 1, created),
('blue_key', 'blue', 4, None),
)
def test_fake_model_property(self):
value = {'bla': [1, 2, {'blub': 'bla'*1000}]}
FM(data=FakeModel(value=value)).put()
self.assertEqual(FM.all()[0].data.value, value)
def db_add(model, key_name, parent=None, **kwargs):
"""
This function creates an object transactionally if it does not exist in
the datastore. Otherwise it returns None.
"""
existing = model.get_by_key_name(key_name, parent=parent)
if not existing:
new_entity = model(parent=parent, key_name=key_name, **kwargs)
new_entity.put()
return new_entity
return None
def cleanup_relations(instance, **kwargs):
if getattr(instance, '__handling_delete', False):
return
rels_seen, to_delete, to_put = get_cleanup_entities(instance)
_get_included_cleanup_entities((instance,), rels_seen, to_delete, to_put)
for entity in [instance] + to_delete:
entity.__handling_delete = True
if to_delete:
db.delete(to_delete)
for entity in [instance] + to_delete:
del entity.__handling_delete
if to_put:
db.put(to_put)
def update(cls, property_name, property_value):
key_name = GlobalPreferences.key_name(property_name)
preference = GlobalPreferences.get_or_insert(key_name, property_name=property_name)
preference.property_value = property_value
# update
db.put(preference)
return preference
def add_or_remove(cls, fingerprint, crash, argv=None, labels=None, is_add=True, delta=1):
# use an issue if one already exists
issue = CrashReport.most_recent_issue(CrashReport.key_name(fingerprint))
key_name = CrashReport.key_name(fingerprint)
config = ShardedCounterConfig.get_sharded_config(key_name)
shards = config.shards
shard_to_use = random.randint(0, shards-1)
shard_key_name = key_name + '_' + str(shard_to_use)
if not argv:
argv = []
crash_report = CrashReport \
.get_or_insert(shard_key_name,
name=key_name,
crash=crash,
fingerprint=fingerprint,
argv=argv,
labels=labels,
issue=issue)
if is_add:
crash_report.count += delta
crash_report.put()
# update caches
memcache.incr(CrashReport.count_cache_key(key_name), delta, initial_value=0)
else:
crash_report.count -= delta
crash_report.put()
memcache.decr(CrashReport.count_cache_key(key_name), delta)
# clear properties cache
CrashReport.clear_properties_cache(key_name)
return crash_report
def update_crash_report(cls, fingerprint, delta_state):
name = CrashReport.key_name(fingerprint)
to_update = list()
q = CrashReport.all()
q.filter('name = ', name)
for crash_report in q.run():
# update state
# only allow * mutable * properties of crash reports to be updated
# having to manually update properties on the entity this was, as expando entities
# do not have a way to update an entity via a property name :(
if 'argv' in delta_state:
crash_report.argv = delta_state.get('argv')
if 'labels' in delta_state:
crash_report.labels = delta_state.get('labels')
if 'date_time' in delta_state:
crash_report.date_time = delta_state.get('date_time')
if 'count' in delta_state:
crash_report.count = delta_state.get('count')
if 'issue' in delta_state:
crash_report.issue = delta_state.get('issue')
if 'state' in delta_state:
crash_report.state = delta_state.get('state')
to_update.append(crash_report)
# update datastore and search indexes
db.put(to_update)
Search.add_crash_reports(to_update)
# clear memcache
CrashReport.clear_properties_cache(name)
# return crash report
return CrashReport.get_crash(fingerprint)
def save_objects(self, keys_to_objects):
cache_keys = [self.key_to_cache_key(object_key) for object_key, this_object in keys_to_objects.iteritems()]
if cache_keys:
existing_objects = FacebookCachedObject.get_by_key_name(cache_keys)
else:
existing_objects = []
db_objects_to_put = []
for obj, (object_key, this_object) in zip(existing_objects, keys_to_objects.iteritems()):
if not self._is_cacheable(object_key, this_object):
#TODO(lambert): cache the fact that it's a private-unshowable event somehow? same as deleted events?
logging.warning("BatchLookup: Looked up id %s but is not cacheable.", object_key)
continue
cache_key = self.key_to_cache_key(object_key)
if not obj:
obj = FacebookCachedObject(key_name=cache_key)
old_json_data = obj.json_data
obj.encode_data(this_object)
if old_json_data != obj.json_data:
self.db_updates += 1
db_objects_to_put.append(obj)
if db_objects_to_put:
try:
db.put(db_objects_to_put)
except apiproxy_errors.CapabilityDisabledError as e:
logging.warning('CapabilityDisabledError: %s', e)
def _run_map_jobs(job_operation_key, backup_info_key, kinds, job_name,
backup_handler, input_reader, output_writer, mapper_params,
mapreduce_params, queue):
"""Creates backup/restore MR jobs for the given operation.
Args:
job_operation_key: a key of utils.DatastoreAdminOperation entity.
backup_info_key: a key of BackupInformation entity.
kinds: a list of kinds to run the M/R for.
job_name: the M/R job name prefix.
backup_handler: M/R job completion handler.
input_reader: M/R input reader.
output_writer: M/R output writer.
mapper_params: custom parameters to pass to mapper.
mapreduce_params: dictionary parameters relevant to the whole job.
queue: the name of the queue that will be used by the M/R.
Returns:
Ids of all started mapper jobs as list of strings.
"""
backup_info = BackupInformation.get(backup_info_key)
if not backup_info:
return []
jobs = utils.RunMapForKinds(
job_operation_key,
kinds,
job_name,
backup_handler,
input_reader,
output_writer,
mapper_params,
mapreduce_params,
queue_name=queue)
backup_info.active_jobs = jobs
backup_info.put(force_writes=True)
return jobs
def finalize_backup_info(backup_info_pk, gcs_path_prefix):
"""Finalize the state of BackupInformation and creates info file for GS."""
def get_backup_info():
return BackupInformation.get(backup_info_pk)
backup_info = db.run_in_transaction(get_backup_info)
if backup_info:
complete_time = datetime.datetime.now()
backup_info.complete_time = complete_time
gs_handle = None
if backup_info.filesystem == FILES_API_GS_FILESYSTEM:
gs_handle = BackupInfoWriter(gcs_path_prefix).write(backup_info)[0]
def set_backup_info_with_finalize_info():
backup_info = get_backup_info()
backup_info.complete_time = complete_time
backup_info.gs_handle = gs_handle
backup_info.put(force_writes=True)
db.run_in_transaction(set_backup_info_with_finalize_info)
logging.info('Backup %s completed', backup_info.name)
else:
logging.warn('Backup %s could not be found', backup_info_pk)
def flush(self):
"""Save aggregated type information to the datastore if changed."""
if self.__needs_save:
def update_aggregation_tx():
aggregation = SchemaAggregationResult.load(
self.__backup_id, self.__kind, self.__shard_id)
if aggregation:
if aggregation.merge(self.__aggregation):
aggregation.put(force_writes=True)
self.__aggregation = aggregation
else:
self.__aggregation.put(force_writes=True)
def mark_aggregation_as_partial_tx():
aggregation = SchemaAggregationResult.load(
self.__backup_id, self.__kind, self.__shard_id)
if aggregation is None:
aggregation = SchemaAggregationResult.create(
self.__backup_id, self.__kind, self.__shard_id)
aggregation.is_partial = True
aggregation.put(force_writes=True)
self.__aggregation = aggregation
try:
db.run_in_transaction(update_aggregation_tx)
except apiproxy_errors.RequestTooLargeError:
db.run_in_transaction(mark_aggregation_as_partial_tx)
self.__needs_save = False
def _drop_gracefully(self):
"""Gracefully drop controller task.
This method is called when decoding controller task payload failed.
Upon this we mark ShardState and MapreduceState as failed so all
tasks can stop.
Writing to datastore is forced (ignore read-only mode) because we
want the tasks to stop badly, and if force_writes was False,
the job would have never been started.
"""
mr_id = self.request.headers[util._MR_ID_TASK_HEADER]
state = model.MapreduceState.get_by_job_id(mr_id)
if not state or not state.active:
return
state.active = False
state.result_status = model.MapreduceState.RESULT_FAILED
config = util.create_datastore_write_config(state.mapreduce_spec)
puts = []
for ss in model.ShardState.find_all_by_mapreduce_state(state):
if ss.active:
ss.set_for_failure()
puts.append(ss)
if len(puts) > model.ShardState._MAX_STATES_IN_MEMORY:
db.put(puts, config=config)
puts = []
db.put(puts, config=config)
self._finalize_job(state.mapreduce_spec, state)
def get(self, d):
BATCH_SIZE = 100
cursor = self.request.get('cursor')
q = Question.all()
if cursor:
q.with_cursor(cursor)
logging.debug("Query count: %d" % q.count())
batch = q.fetch(BATCH_SIZE)
identifiers = []
if batch:
new_cursor = q.cursor()
next_url = '/admin/schema/do_schema?cursor=%s' % (new_cursor)
changed = []
for item in batch:
edited = False
try:
if item.correct_response:
item.correct_responses=[item.correct_response]
item.correct_response = None
edited = True
if edited:
changed.append(item)
except Exception, e:
logging.error("Error: %s" % e)
identifiers.append("Question: %s, Changed: %s" % (item.correct_responses, edited))
db.put(changed)
context = {
'identifiers': identifiers,
'next_url': next_url,
}
self.render_template("schema_update.html", **context)
else:
self.redirect("/admin")
def update(self, tablename, query, update_fields):
# self.db['_lastsql'] = self._update(tablename, query, update_fields)
(items, tablename, fields) = self.select_raw(query)
counter = 0
for item in items:
for field, value in update_fields:
setattr(item, field.name, self.represent(value, field.type))
item.put()
counter += 1
LOGGER.info(str(counter))
return counter
def insert(self, table, fields):
dfields = dict((f.name, self.represent(v, f.type)) for f, v in fields)
# table._db['_lastsql'] = self._insert(table, fields)
tmp = table._tableobj(**dfields)
tmp.put()
key = tmp.key if self.use_ndb else tmp.key()
rid = Reference(key.id())
(rid._table, rid._record, rid._gaekey) = (table, None, key)
return rid
def bulk_insert(self, table, items):
parsed_items = []
for item in items:
dfields = dict((f.name, self.represent(v, f.type)) for f, v in item)
parsed_items.append(table._tableobj(**dfields))
if self.use_ndb:
ndb.put_multi(parsed_items)
else:
gae.put(parsed_items)
return True
def post(self):
"""Handler for post requests to datastore_admin/import_backup.do.
Import is executed and user is redirected to the base-path handler.
"""
gs_handle = self.request.get('gs_handle')
token = self.request.get('xsrf_token')
error = None
if gs_handle and utils.ValidateXsrfToken(token, XSRF_ACTION):
try:
bucket_name, path = parse_gs_handle(gs_handle)
file_content = get_gs_object(bucket_name, path)
entities = parse_backup_info_file(file_content)
original_backup_info = entities.next()
entity = datastore.Entity(BackupInformation.kind())
entity.update(original_backup_info)
backup_info = BackupInformation.from_entity(entity)
if original_backup_info.key().app() != os.getenv('APPLICATION_ID'):
backup_info.original_app = original_backup_info.key().app()
def tx():
backup_info.put(force_writes=True)
kind_files_models = []
for entity in entities:
kind_files = backup_info.create_kind_backup_files(
entity.key().name(), entity['files'])
kind_files_models.append(kind_files)
db.put(kind_files_models, force_writes=True)
db.run_in_transaction(tx)
backup_id = str(backup_info.key())
except Exception, e:
logging.exception('Failed to Import datastore backup information.')
error = e.message
if error:
self.SendRedirect(params=[('error', error)])
elif self.request.get('Restore'):
ConfirmRestoreFromBackupHandler.Render(
self, default_backup_id=backup_id,
default_delete_backup_after_restore=True)
else:
self.SendRedirect()
def _try_free_lease(self, shard_state, slice_retry=False):
"""Try to free lease.
A lightweight transaction to update shard_state and unset
slice_start_time to allow the next retry to happen without blocking.
We don't care if this fails or not because the lease will expire
anyway.
Under normal execution, _save_state_and_schedule_next is the exit point.
It updates/saves shard state and schedules the next slice or returns.
Other exit points are:
1. _are_states_consistent: at the beginning of handle, checks
if datastore states and the task are in sync.
If not, raise or return.
2. _attempt_slice_retry: may raise exception to taskqueue.
3. _save_state_and_schedule_next: may raise exception when taskqueue/db
unreachable.
This handler should try to free the lease on every exceptional exit point.
Args:
shard_state: model.ShardState.
slice_retry: whether to count this as a failed slice execution.
"""
@db.transactional
def _tx():
fresh_state = model.ShardState.get_by_shard_id(shard_state.shard_id)
if fresh_state and fresh_state.active:
fresh_state.slice_start_time = None
fresh_state.slice_request_id = None
if slice_retry:
fresh_state.slice_retries += 1
fresh_state.put()
try:
_tx()
except Exception, e:
logging.warning(e)
logging.warning(
"Release lock for shard %s failed. Wait for lease to expire.",
shard_state.shard_id)
def _finalize_job(cls, mapreduce_spec, mapreduce_state):
"""Finalize job execution.
Invokes done callback and save mapreduce state in a transaction,
and schedule necessary clean ups. This method is idempotent.
Args:
mapreduce_spec: an instance of MapreduceSpec
mapreduce_state: an instance of MapreduceState
"""
config = util.create_datastore_write_config(mapreduce_spec)
queue_name = util.get_queue_name(mapreduce_spec.params.get(
model.MapreduceSpec.PARAM_DONE_CALLBACK_QUEUE))
done_callback = mapreduce_spec.params.get(
model.MapreduceSpec.PARAM_DONE_CALLBACK)
done_callback_target = mapreduce_spec.params.get(
model.MapreduceSpec.PARAM_DONE_CALLBACK_TARGET)
done_task = None
if done_callback:
headers = util._get_task_headers(
mapreduce_spec.mapreduce_id,
util.CALLBACK_MR_ID_TASK_HEADER,
set_host_header=(done_callback_target is None))
done_task = taskqueue.Task(
url=done_callback,
target=done_callback_target,
headers=headers,
method=mapreduce_spec.params.get("done_callback_method", "POST"))
@db.transactional(retries=5)
def _put_state():
"""Helper to store state."""
fresh_state = model.MapreduceState.get_by_job_id(
mapreduce_spec.mapreduce_id)
if not fresh_state.active:
logging.warning(
"Job %s is not active. Looks like spurious task execution. "
"Dropping task.", mapreduce_spec.mapreduce_id)
return
mapreduce_state.put(config=config)
if done_task and not _run_task_hook(
mapreduce_spec.get_hooks(),
"enqueue_done_task",
done_task,
queue_name,
transactional=True):
done_task.add(queue_name, transactional=True)
_put_state()
logging.info("Final result for job '%s' is '%s'",
mapreduce_spec.mapreduce_id, mapreduce_state.result_status)
cls._clean_up_mr(mapreduce_spec)
