def find_lost(timeout):
from yawn.task.models import Execution
# Make a sparse index so looking up active workers is fast:
# CREATE INDEX yawn_worker_active ON yawn_worker (status) WHERE status = 'active'
lost = Worker.objects.filter(
status=Worker.ACTIVE, last_heartbeat__lt=functions.Now() - timedelta(seconds=timeout)
)
for worker in lost:
logger.warning('Marking %r as lost', worker)
worker.status = Worker.LOST
worker.save()
executions = worker.execution_set.filter(status=Execution.RUNNING)
for execution in executions:
logger.warning('Marking %r as lost', execution)
execution.mark_finished(lost=True)
python类Now()的实例源码
def submit_run(self, parameters=None, scheduled_time=None):
"""Create a run of this template"""
from yawn.task.models import Task
run_parameters = self.parameters.copy()
run_parameters.update(parameters or {})
run = Run.objects.create(
workflow=self,
submitted_time=functions.Now(),
scheduled_time=scheduled_time,
parameters=run_parameters,
)
for template in self.template_set.all():
task = Task.objects.create(
run=run,
template=template,
)
if not template.upstream.exists():
task.enqueue()
return run
def __init__(self, output_field=None, **extra):
if output_field is None:
output_field = DateTimeField()
super(Now, self).__init__(output_field=output_field, **extra)
def update_worker(self):
"""
Look for executors where the connection has broken and tasks need to be re-submitted.
"""
if not self.worker:
self.worker = Worker.objects.create(
name=self.name,
start_timestamp=functions.Now(),
last_heartbeat=functions.Now()
)
else:
self.worker.refresh_from_db()
if self.worker.status == Worker.LOST:
# someone else marked us as lost, terminate all tasks and exit
logger.warning('Marked as lost, committing harakiri')
self.state = State.terminate
self.executor.mark_terminated(self.executor.get_running_ids())
return
# update our timestamp so no one marks us as lost
self.worker.last_heartbeat = functions.Now()
self.worker.save()
# look for lost workers and re-queue their tasks
Worker.find_lost(self.timeout)
def mark_finished(self, exit_code=None, lost=False):
"""
Update the execution status after it has finished:
successfully, in error, or because it was lost.
Also update the task and workflow; re-queue the task if
it should be retried.
"""
if lost:
self.status = Execution.LOST
self.task.enqueue()
elif exit_code == 0:
self.task.status = Task.SUCCEEDED
self.status = Execution.SUCCEEDED
else:
# queue another run if there are remaining retries
# (current execution is not in count b/c it hasn't been saved yet)
failed_count = self.task.execution_set.filter(status=Task.FAILED).count()
if failed_count < self.task.template.max_retries:
self.task.enqueue()
else:
self.task.status = Task.FAILED
self.status = Execution.FAILED
if self.task.status != Task.RUNNING:
self.task.save()
with transaction.atomic():
self.task.update_downstream()
if self.task.run:
self.task.run.update_status()
self.stop_timestamp = functions.Now()
# need to be careful not to overwrite stdout/stderr
self.save(update_fields=['status', 'stop_timestamp'])
def reschedule_scheduled_jobs(self):
ScheduledJob = self.get_model('ScheduledJob')
jobs = ScheduledJob.objects.filter(
enabled=True, scheduled_time__lte=Now())
self.reschedule_jobs(jobs)
def send_or_update(self, user, template, **context):
"""
Update or send a notice.
The notices to update are located by `target`, `target_slug`, `actor`,
which are ALL required to be specified.
"""
filter_fields = set(context.get('filter_fields', {'target', 'target_slug', 'actor', 'user', 'category'}))
assert filter_fields, '`filter_fields` should not be empty.'
specified = {'user', 'category', *context}
missing = filter_fields - specified
assert not missing, 'Members %s of `filter_fields` are missing.' % ', '.join(missing)
keywords = {
name: value
for name, value in {
'user': user, 'template': template, 'category': self.category,
**context
}.items()
if name in filter_fields
}
if 'target' in filter_fields and context['target'] is not None:
target = context['target']
ctype = ContentType.objects.get_for_model(target)
keywords.pop('target')
keywords.update({
'target_type': ctype,
'target_id': target.pk
})
try_find = Notice.objects.filter(**keywords)
if try_find.count():
message = render_notice_message(
template, self, user=user, **context)
try_find.update(message=message, has_read=False, created=Now())
notice = None
else:
notice = self.send(user, template, **context)
Notice.objects.broadcast_stats([user])
return notice