# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Library for frontend.afe.models.JobHandoff and job cleanup."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import datetime
import logging
import socket
from lucifer import autotest
logger = logging.getLogger(__name__)
_JOB_GRACE_SECS = 10
def incomplete():
"""Return a QuerySet of incomplete JobHandoffs.
JobHandoff created within a cutoff period are exempt to allow the
job the chance to acquire its lease file; otherwise, incomplete jobs
without an active lease are considered dead.
@returns: Django QuerySet
"""
models = autotest.load('frontend.afe.models')
Q = autotest.deps_load('django.db.models').Q
# Time ---*---------|---------*-------|--->
# incomplete cutoff newborn now
cutoff = (datetime.datetime.now()
- datetime.timedelta(seconds=_JOB_GRACE_SECS))
return (models.JobHandoff.objects
.filter(completed=False, created__lt=cutoff)
.filter(Q(drone=socket.gethostname()) | Q(drone=None)))
def clean_up(job_ids):
"""Clean up failed jobs failed in database.
This brings the database into a clean state, which includes marking
the job, HQEs, and hosts.
"""
if not job_ids:
return
models = autotest.load('frontend.afe.models')
logger.info('Cleaning up failed jobs: %r', job_ids)
hqes = models.HostQueueEntry.objects.filter(job_id__in=job_ids)
logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
_clean_up_hqes(hqes)
host_ids = {id for id in hqes.values_list('host_id', flat=True)
if id is not None}
logger.debug('Found Hosts associated with jobs: %r', host_ids)
_clean_up_hosts(host_ids)
def _clean_up_hqes(hqes):
models = autotest.load('frontend.afe.models')
logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
hqes.update(complete=True,
active=False,
status=models.HostQueueEntry.Status.FAILED)
(hqes.exclude(started_on=None)
.update(finished_on=datetime.datetime.now()))
def _clean_up_hosts(host_ids):
models = autotest.load('frontend.afe.models')
transaction = autotest.deps_load('django.db.transaction')
with transaction.commit_on_success():
active_hosts = {
id for id in (models.HostQueueEntry.objects
.filter(active=True, complete=False)
.values_list('host_id', flat=True))
if id is not None}
logger.debug('Found active Hosts: %r', active_hosts)
(models.Host.objects
.filter(id__in=host_ids)
.exclude(id__in=active_hosts)
.update(status=None))
def mark_complete(job_ids):
"""Mark the corresponding JobHandoffs as completed."""
if not job_ids:
return
models = autotest.load('frontend.afe.models')
logger.info('Marking job handoffs complete: %r', job_ids)
(models.JobHandoff.objects
.filter(job_id__in=job_ids)
.update(completed=True))