#!/usr/bin/python
import sys, optparse, pwd
import common
from autotest_lib.cli import rpc, host
from autotest_lib.client.common_lib import host_queue_entry_states
parser = optparse.OptionParser(
usage='Usage: %prog [options] <job id> [<hostname>]\n\n'
'Describes why the given job on the given host has not started.')
parser.add_option('-w', '--web',
help='Autotest server to use (i.e. "autotest")')
options, args = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(1)
job_id = int(args[0])
autotest_host = rpc.get_autotest_server(options.web)
proxy = rpc.afe_comm(autotest_host)
# job exists?
jobs = proxy.run('get_jobs', id=job_id)
if not jobs:
print 'No such job', job_id
sys.exit(1)
job = jobs[0]
owner = job['owner']
RUNNING_HQE_STATUSES = host_queue_entry_states.ACTIVE_STATUSES
# any entry eligible for this host?
queue_entries = proxy.run('get_host_queue_entries', job__id=job_id)
### Divine why an atomic group job is or is not running.
if queue_entries and queue_entries[0]['atomic_group']:
if queue_entries[0]['status'] in RUNNING_HQE_STATUSES:
print 'Job %d appears to have started (status: %s).' % (
job_id, queue_entries[0]['status'])
sys.exit(0)
# Hosts in Repairing or Repair Failed will have Queued queue entries.
# We shouldn't consider those queue entries as a multi-group job.
repair_hostnames = []
for queue_entry in queue_entries:
if queue_entry['host'] and queue_entry['host']['status']:
if queue_entry['host']['status'].startswith('Repair'):
repair_hostnames.append(queue_entry['host']['hostname'])
if queue_entry['status'] in ('Completed', 'Stopped'):
print 'This job has already finished.'
sys.exit(0)
queue_entries_with_hosts = [queue_entry for queue_entry in queue_entries
if queue_entry['host']]
all_queue_entries_have_hosts = (len(queue_entries) ==
len(queue_entries_with_hosts))
if (not all_queue_entries_have_hosts and len(queue_entries) > 1 and
not repair_hostnames):
# We test repair_hostnames so that this message is not printed when
# the script is run on an atomic group job which has hosts assigned
# but is not running because too many of them are in Repairing or will
# never run because hosts have exited Repairing into the Repair Failed
# dead end.
print 'This script does not support multi-group atomic group jobs.'
print
print 'Jobs scheduled in that state are typically unintentional.'
print
print 'Did you perhaps schedule the job via the web frontend and ask'
print 'that it run on more than 1 (atomic group) of hosts via the '
print '"Run on any" box? If so, always enter 1 there when scheduling'
print 'jobs on anything marked "(atomic group)".'
print
print len(queue_entries), 'non-started atomic group HostQueueEntries',
print 'found for job', job_id
sys.exit(1)
atomic_group_name = queue_entries[0]['atomic_group']['name']
# Get the list of labels associated with this atomic group.
atomic_labels = proxy.run('get_labels',
atomic_group__name=atomic_group_name)
if len(atomic_labels) < 1:
print 'Job requests atomic group %s but no labels' % atomic_group_name
print '(and thus no hosts) are associated with that atomic group.'
job_sync_count = job['synch_count']
# Ugh! This is returned as a comma separated str of label names.
if job.get('dependencies'):
job_dependency_label_names = job['dependencies'].split(',')
else:
job_dependency_label_names = []
meta_host_name = queue_entries[0]['meta_host']
if meta_host_name:
meta_host = proxy.run('get_labels', atomic_group__name=meta_host_name)[0]
else:
meta_host = None
# A mapping from label name -> a list of hostnames usable for this job.
runnable_atomic_label_names = {}
# A mapping from label name -> a host_exclude_reasons map as described
# within the loop below. Any atomic group labels in this map are not
# ready to run the job for the reasons contained within.
atomic_label_exclude_reasons = {}
for label in atomic_labels:
label_name = label['name']
if meta_host and meta_host_name != label_name:
print 'Cannot run on atomic label %s due to meta_host %s.' % (
label_name, meta_host_name)
continue
for dep_name in job_dependency_label_names:
if dep_name != label_name:
print 'Not checking hosts in atomic label %s against' % (
label_name,)
print 'job dependency label %s. There may be less hosts' % (
dep_name,)
print 'than examined below available to run this job.'
# Get the list of hosts associated with this atomic group label.
atomic_hosts = proxy.run('get_hosts', multiple_labels=[label_name])
# A map of hostname -> A list of reasons it can't be used.
host_exclude_reasons = {}
atomic_hostnames = [h['hostname'] for h in atomic_hosts]
# Map hostnames to a list of ACL names on that host.
acl_groups = proxy.run('get_acl_groups',
hosts__hostname__in=atomic_hostnames)
hostname_to_acl_name_list = {}
for acl in acl_groups:
for hostname in acl['hosts']:
hostname_to_acl_name_list.setdefault(hostname, []).append(
acl['name'])
# Exclude any hosts that ACLs deny us access to.
accessible_hosts = proxy.run('get_hosts', hostname__in=atomic_hostnames,
aclgroup__users__login=owner)
assert len(accessible_hosts) <= len(atomic_hosts)
if len(accessible_hosts) != len(atomic_hosts):
accessible_hostnames = set(h['hostname'] for h in accessible_hosts)
acl_excluded_hostnames = (set(atomic_hostnames) -
accessible_hostnames)
for hostname in acl_excluded_hostnames:
acls = ','.join(hostname_to_acl_name_list[hostname])
host_exclude_reasons.setdefault(hostname, []).append(
'User %s does not have ACL access. ACLs: %s' % (
owner, acls))
# Check for locked hosts.
locked_hosts = [h for h in atomic_hosts if h['locked']]
for host in locked_hosts:
locker = host.get('locked_by') or 'UNKNOWN'
msg = 'Locked by user %s on %s. No jobs will schedule on it.' % (
locker, host.get('lock_time'))
host_exclude_reasons.setdefault(host['hostname'], []).append(msg)
# Exclude hosts that are not Ready.
for host in atomic_hosts:
hostname = host['hostname']
if host['status'] != 'Ready':
message = 'Status is %s' % host['status']
if host['status'] in ('Verifying', 'Pending', 'Running'):
running_hqes = proxy.run(
'get_host_queue_entries', host__hostname=hostname,
status__in=RUNNING_HQE_STATUSES)
if not running_hqes:
message += ' (unknown job)'
else:
message += ' (job %d)' % running_hqes[0]['job']['id']
host_exclude_reasons.setdefault(hostname, []).append(message)
# If we don't have enough usable hosts, this group cannot run the job.
usable_hostnames = [host['hostname'] for host in atomic_hosts
if host['hostname'] not in host_exclude_reasons]
if len(usable_hostnames) < job_sync_count:
message = ('%d hosts are required but only %d available.' %
(job_sync_count, len(usable_hostnames)))
atomic_label_exclude_reasons[label_name] = (message,
host_exclude_reasons)
else:
runnable_atomic_label_names[label_name] = usable_hostnames
for label_name, reason_tuple in atomic_label_exclude_reasons.iteritems():
job_reason, hosts_reasons = reason_tuple
print 'Atomic group "%s" via label "%s" CANNOT run job %d because:' % (
atomic_group_name, label_name, job_id)
print job_reason
for hostname in sorted(hosts_reasons.keys()):
for reason in hosts_reasons[hostname]:
print '%s\t%s' % (hostname, reason)
print
for label_name, host_list in runnable_atomic_label_names.iteritems():
print 'Atomic group "%s" via label "%s" is READY to run job %d on:' % (
atomic_group_name, label_name, job_id)
print ', '.join(host_list)
print 'Is the job scheduler healthy?'
print
sys.exit(0)
### Not an atomic group synchronous job:
if len(args) != 2:
if len(queue_entries) == 1 and queue_entries[0]['host']:
hostname = queue_entries[0]['host']['hostname']
else:
parser.print_help()
print '\nERROR: A hostname associated with the job is required.'
sys.exit(1)
else:
hostname = args[1]
# host exists?
hosts = proxy.run('get_hosts', hostname=hostname)
if not hosts:
print 'No such host', hostname
sys.exit(1)
host = hosts[0]
# Boolean to track our findings. We want to list all reasons it won't run,
# not just the first.
job_will_run = True
entries_for_this_host = [entry for entry in queue_entries
if entry['host']
and entry['host']['hostname'] == hostname]
host_label_names = set(host['labels'])
eligible_metahost_entries = [entry for entry in queue_entries
if entry['meta_host'] and not entry['host']
and entry['meta_host'] in host_label_names
and not entry['complete']]
if entries_for_this_host:
assert len(entries_for_this_host) == 1, (
'Multiple entries for this job assigned to this host!')
entry = entries_for_this_host[0]
if entry['active'] or entry['complete']:
print ('Job already ran or is running on this host! (status: %s)' %
entry['full_status'])
sys.exit(0)
is_metahost = False
else:
# no entry for this host -- maybe an eligible metahost entry?
if not eligible_metahost_entries:
print ("Host isn't scheduled for this job, and no eligible metahost "
"entry exists")
sys.exit(0)
is_metahost = True
# meets atomic group requirements?
host_labels = proxy.run('get_labels', name__in=list(host_label_names))
host_atomic_group_labels = [label for label in host_labels
if label['atomic_group']]
host_atomic_group_name = None
if host_atomic_group_labels:
atomic_groups = set()
for label in host_atomic_group_labels:
atomic_groups.add(label['atomic_group']['name'])
if len(atomic_groups) != 1:
print 'Host has more than one atomic group!'
print list(atomic_groups)
sys.exit(1)
host_atomic_group_label = host_atomic_group_labels[0]
host_atomic_group_name = host_atomic_group_label['atomic_group']['name']
job_atomic_groups = set(entry['atomic_group'] for entry in queue_entries)
assert len(job_atomic_groups) == 1, 'Job has more than one atomic group value!'
job_atomic_group = job_atomic_groups.pop() # might be None
job_atomic_group_name = None
if job_atomic_group:
job_atomic_group_name = job_atomic_group['name']
if host_atomic_group_name != job_atomic_group_name:
print ('Job is for atomic group %s, but host is in atomic group %s '
'(label %s)' %
(job_atomic_group_name, host_atomic_group_name,
host_atomic_group_label['name']))
job_will_run = False
# host locked?
if host['locked']:
print 'Host is locked by', host['locked_by'], 'no jobs will schedule on it.'
job_will_run = False
# acl accessible?
accessible = proxy.run('get_hosts', hostname=hostname,
aclgroup__users__login=owner)
if not accessible:
host_acls = ', '.join(group['name'] for group in
proxy.run('get_acl_groups', hosts__hostname=hostname))
owner_acls = ', '.join(group['name'] for group in
proxy.run('get_acl_groups', users__login=owner))
print 'Host not ACL-accessible to job owner', owner
print ' Host ACLs:', host_acls
print ' Owner Acls:', owner_acls
job_will_run = False
# meets dependencies?
job_deps_list = job['dependencies'].split(',')
job_deps = set()
if job_deps_list != ['']:
job_deps = set(job_deps_list)
unmet = job_deps - host_label_names
if unmet:
print ("Host labels (%s) don't satisfy job dependencies: %s" %
(', '.join(host_label_names), ', '.join(unmet)))
job_will_run = False
# at this point, if the job is for an unassigned atomic group, things are too
# complicated to proceed
unassigned_atomic_group_entries = [entry for entry in queue_entries
if entry['atomic_group']
and not entry['host']]
if unassigned_atomic_group_entries:
print ("Job is for an unassigned atomic group. That's too complicated, I "
"can't give you any definite answers. Sorry.")
sys.exit(1)
# meets only_if_needed labels?
if is_metahost:
metahost_names = set(entry['meta_host']
for entry in eligible_metahost_entries)
job_deps_and_metahosts = job_deps.union(metahost_names)
for label in host_labels:
unmet_exclusive_label = (label['only_if_needed'] and
label['name'] not in job_deps_and_metahosts)
if unmet_exclusive_label:
print ('Host contains "only if needed" label %s, unused by job '
'dependencies and metahosts' % label['name'])
job_will_run = False
# host ready?
if host['status'] != 'Ready':
if host['status'] == 'Pending':
active = proxy.run('get_host_queue_entries',
host=host['id'], active=True)
if not active:
print ('Host %s seems to be in "Pending" state incorrectly; please '
'report this to the Autotest team' % hostname)
sys.exit(1)
print 'Host not in "Ready" status (status="%s")' % host['status']
job_will_run = False
if job_will_run:
print ("Job %s should run on host %s; if you've already waited about ten "
"minutes or longer, it's probably a server issue or a bug." %
(job_id, hostname))
sys.exit(1)
else:
print "All of the reasons this job is not running are listed above."
sys.exit(0)