# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This file contains utility functions for host_history.
import collections
import copy
import multiprocessing.pool
from itertools import groupby
import common
from autotest_lib.client.common_lib import time_utils
from autotest_lib.client.common_lib.cros.graphite import autotest_es
from autotest_lib.frontend import setup_django_environment
from autotest_lib.frontend.afe import models
from autotest_lib.site_utils import host_label_utils
from autotest_lib.site_utils import job_history
_HOST_HISTORY_TYPE = 'host_history'
_LOCK_HISTORY_TYPE = 'lock_history'
# The maximum number of days that the script will lookup for history.
_MAX_DAYS_FOR_HISTORY = 90
class NoHostFoundException(Exception):
"""Exception raised when no host is found to search for history.
"""
def get_matched_hosts(board, pool):
"""Get duts with matching board and pool labels from metaDB.
@param board: board of DUT, set to None if board doesn't need to match.
@param pool: pool of DUT, set to None if pool doesn't need to match.
@return: A list of duts that match the specified board and pool.
"""
labels = []
if pool:
labels.append('pool:%s' % pool)
if board:
labels.append('board:%s' % board)
host_labels = host_label_utils.get_host_labels(labels=labels)
return host_labels.keys()
def prepopulate_dict(keys, value, extras=None):
"""Creates a dictionary with val=value for each key.
@param keys: list of keys
@param value: the value of each entry in the dict.
@param extras: list of additional keys
@returns: dictionary
"""
result = collections.OrderedDict()
extra_keys = tuple(extras if extras else [])
for key in keys + extra_keys:
result[key] = value
return result
def lock_history_to_intervals(initial_lock_val, t_start, t_end, lock_history):
"""Converts lock history into a list of intervals of locked times.
@param initial_lock_val: Initial value of the lock (False or True)
@param t_start: beginning of the time period we are interested in.
@param t_end: end of the time period we are interested in.
@param lock_history: Result of querying es for locks (dict)
This dictionary should contain keys 'locked' and 'time_recorded'
@returns: Returns a list of tuples where the elements of each tuples
represent beginning and end of intervals of locked, respectively.
"""
locked_intervals = []
t_prev = t_start
state_prev = initial_lock_val
for entry in lock_history.hits:
t_curr = entry['time_recorded']
#If it is locked, then we put into locked_intervals
if state_prev:
locked_intervals.append((t_prev, t_curr))
# update vars
t_prev = t_curr
state_prev = entry['locked']
if state_prev:
locked_intervals.append((t_prev, t_end))
return locked_intervals
def find_most_recent_entry_before(t, type_str, hostname, fields):
"""Returns the fields of the most recent entry before t.
@param t: time we are interested in.
@param type_str: _type in esdb, such as 'host_history' (string)
@param hostname: hostname of DUT (string)
@param fields: list of fields we are interested in
@returns: time, field_value of the latest entry.
"""
# History older than 90 days are ignored. This helps the ES query faster.
t_epoch = time_utils.to_epoch_time(t)
result = autotest_es.query(
fields_returned=fields,
equality_constraints=[('_type', type_str),
('hostname', hostname)],
range_constraints=[('time_recorded',
t_epoch-3600*24*_MAX_DAYS_FOR_HISTORY, t_epoch)],
size=1,
sort_specs=[{'time_recorded': 'desc'}])
if result.total > 0:
return result.hits[0]
return {}
def get_host_history_intervals(input):
"""Gets stats for a host.
This method uses intervals found in metaDB to build a full history of the
host. The intervals argument contains a list of metadata from querying ES
for records between t_start and t_end. To get the status from t_start to
the first record logged in ES, we need to look back to the last record
logged in ES before t_start.
@param input: A dictionary of input args, which including following args:
t_start: beginning of time period we are interested in.
t_end: end of time period we are interested in.
hostname: hostname for the host we are interested in (string)
intervals: intervals from ES query.
@returns: dictionary, num_entries_found
dictionary of status: time spent in that status
num_entries_found: number of host history entries
found in [t_start, t_end]
"""
t_start = input['t_start']
t_end = input['t_end']
hostname = input['hostname']
intervals = input['intervals']
lock_history_recent = find_most_recent_entry_before(
t=t_start, type_str=_LOCK_HISTORY_TYPE, hostname=hostname,
fields=['time_recorded', 'locked'])
# I use [0] and [None] because lock_history_recent's type is list.
t_lock = lock_history_recent.get('time_recorded', None)
t_lock_val = lock_history_recent.get('locked', None)
t_metadata = find_most_recent_entry_before(
t=t_start, type_str=_HOST_HISTORY_TYPE, hostname=hostname,
fields=None)
t_host = t_metadata.pop('time_recorded', None)
t_host_stat = t_metadata.pop('status', None)
status_first = t_host_stat if t_host else 'Ready'
t = min([t for t in [t_lock, t_host, t_start] if t])
t_epoch = time_utils.to_epoch_time(t)
t_end_epoch = time_utils.to_epoch_time(t_end)
lock_history_entries = autotest_es.query(
fields_returned=['locked', 'time_recorded'],
equality_constraints=[('_type', _LOCK_HISTORY_TYPE),
('hostname', hostname)],
range_constraints=[('time_recorded', t_epoch, t_end_epoch)],
sort_specs=[{'time_recorded': 'asc'}])
# Validate lock history. If an unlock event failed to be recorded in metadb,
# lock history will show the dut being locked while host still has status
# changed over the time. This check tries to remove the lock event in lock
# history if:
# 1. There is only one entry in lock_history_entries (it's a good enough
# assumption to avoid the code being over complicated.
# 2. The host status has changes after the lock history starts as locked.
if (len(lock_history_entries.hits) == 1 and t_lock_val and
len(intervals) >1):
locked_intervals = None
print ('Lock history of dut %s is ignored, the dut may have missing '
'data in lock history in metadb. Try to lock and unlock the dut '
'in AFE will force the lock history to be updated in metadb.'
% hostname)
else:
locked_intervals = lock_history_to_intervals(t_lock_val, t, t_end,
lock_history_entries)
num_entries_found = len(intervals)
t_prev = t_start
status_prev = status_first
metadata_prev = t_metadata
intervals_of_statuses = collections.OrderedDict()
for entry in intervals:
metadata = entry.copy()
t_curr = metadata.pop('time_recorded')
status_curr = metadata.pop('status')
intervals_of_statuses.update(calculate_status_times(
t_prev, t_curr, status_prev, metadata_prev, locked_intervals))
# Update vars
t_prev = t_curr
status_prev = status_curr
metadata_prev = metadata
# Do final as well.
intervals_of_statuses.update(calculate_status_times(
t_prev, t_end, status_prev, metadata_prev, locked_intervals))
return hostname, intervals_of_statuses, num_entries_found
def calculate_total_times(intervals_of_statuses):
"""Calculates total times in each status.
@param intervals_of_statuses: ordereddict where key=(ti, tf) and val=status
@returns: dictionary where key=status value=time spent in that status
"""
total_times = prepopulate_dict(models.Host.Status.names, 0.0,
extras=['Locked'])
for key, status_info in intervals_of_statuses.iteritems():
ti, tf = key
total_times[status_info['status']] += tf - ti
return total_times
def aggregate_hosts(intervals_of_statuses_list):
"""Aggregates history of multiple hosts
@param intervals_of_statuses_list: A list of dictionaries where keys
are tuple (ti, tf), and value is the status along with other metadata.
@returns: A dictionary where keys are strings, e.g. 'status' and
value is total time spent in that status among all hosts.
"""
stats_all = prepopulate_dict(models.Host.Status.names, 0.0,
extras=['Locked'])
num_hosts = len(intervals_of_statuses_list)
for intervals_of_statuses in intervals_of_statuses_list:
total_times = calculate_total_times(intervals_of_statuses)
for status, delta in total_times.iteritems():
stats_all[status] += delta
return stats_all, num_hosts
def get_stats_string_aggregate(labels, t_start, t_end, aggregated_stats,
num_hosts):
"""Returns string reporting overall host history for a group of hosts.
@param labels: A list of labels useful for describing the group
of hosts these overall stats represent.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param aggregated_stats: A dictionary where keys are string, e.g. 'status'
value is total time spent in that status among all hosts.
@returns: string representing the aggregate stats report.
"""
result = 'Overall stats for hosts: %s \n' % (', '.join(labels))
result += ' %s - %s \n' % (time_utils.epoch_time_to_date_string(t_start),
time_utils.epoch_time_to_date_string(t_end))
result += ' Number of total hosts: %s \n' % (num_hosts)
# This is multiplied by time_spent to get percentage_spent
multiplication_factor = 100.0 / ((t_end - t_start) * num_hosts)
for status, time_spent in aggregated_stats.iteritems():
# Normalize by the total time we are interested in among ALL hosts.
spaces = ' ' * (15 - len(status))
percent_spent = multiplication_factor * time_spent
result += ' %s: %s %.2f %%\n' % (status, spaces, percent_spent)
result += '- -- --- ---- ----- ---- --- -- -\n'
return result
def get_overall_report(label, t_start, t_end, intervals_of_statuses_list):
"""Returns string reporting overall host history for a group of hosts.
@param label: A string that can be useful for showing what type group
of hosts these overall stats represent.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param intervals_of_statuses_list: A list of dictionaries where keys
are tuple (ti, tf), and value is the status along with other metadata,
e.g., task_id, task_name, job_id etc.
"""
stats_all, num_hosts = aggregate_hosts(
intervals_of_statuses_list)
return get_stats_string_aggregate(
label, t_start, t_end, stats_all, num_hosts)
def get_intervals_for_host(t_start, t_end, hostname):
"""Gets intervals for the given.
Query metaDB to return all intervals between given start and end time.
Note that intervals found in metaDB may miss the history from t_start to
the first interval found.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hosts: A list of hostnames to look for history.
@param board: Name of the board to look for history. Default is None.
@param pool: Name of the pool to look for history. Default is None.
@returns: A dictionary of hostname: intervals.
"""
t_start_epoch = time_utils.to_epoch_time(t_start)
t_end_epoch = time_utils.to_epoch_time(t_end)
host_history_entries = autotest_es.query(
fields_returned=None,
equality_constraints=[('_type', _HOST_HISTORY_TYPE),
('hostname', hostname)],
range_constraints=[('time_recorded', t_start_epoch,
t_end_epoch)],
sort_specs=[{'time_recorded': 'asc'}])
return host_history_entries.hits
def get_intervals_for_hosts(t_start, t_end, hosts=None, board=None, pool=None):
"""Gets intervals for given hosts or board/pool.
Query metaDB to return all intervals between given start and end time.
If a list of hosts is provided, the board and pool constraints are ignored.
If hosts is set to None, and board or pool is set, this method will attempt
to search host history with labels for all hosts, to help the search perform
faster.
If hosts, board and pool are all set to None, return intervals for all
hosts.
Note that intervals found in metaDB may miss the history from t_start to
the first interval found.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hosts: A list of hostnames to look for history.
@param board: Name of the board to look for history. Default is None.
@param pool: Name of the pool to look for history. Default is None.
@returns: A dictionary of hostname: intervals.
"""
hosts_intervals = {}
if hosts:
for host in hosts:
hosts_intervals[host] = get_intervals_for_host(t_start, t_end, host)
else:
hosts = get_matched_hosts(board, pool)
if not hosts:
raise NoHostFoundException('No host is found for board:%s, pool:%s.'
% (board, pool))
equality_constraints=[('_type', _HOST_HISTORY_TYPE),]
if board:
equality_constraints.append(('labels', 'board:'+board))
if pool:
equality_constraints.append(('labels', 'pool:'+pool))
t_start_epoch = time_utils.to_epoch_time(t_start)
t_end_epoch = time_utils.to_epoch_time(t_end)
results = autotest_es.query(
equality_constraints=equality_constraints,
range_constraints=[('time_recorded', t_start_epoch,
t_end_epoch)],
sort_specs=[{'hostname': 'asc'}])
results_group_by_host = {}
for hostname,intervals_for_host in groupby(results.hits,
lambda h: h['hostname']):
results_group_by_host[hostname] = intervals_for_host
for host in hosts:
intervals = results_group_by_host.get(host, None)
# In case the host's board or pool label was modified after
# the last status change event was reported, we need to run a
# separate query to get its history. That way the host's
# history won't be shown as blank.
if not intervals:
intervals = get_intervals_for_host(t_start, t_end, host)
hosts_intervals[host] = intervals
return hosts_intervals
def get_report(t_start, t_end, hosts=None, board=None, pool=None,
print_each_interval=False):
"""Gets history for given hosts or board/pool
If a list of hosts is provided, the board and pool constraints are ignored.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hosts: A list of hostnames to look for history.
@param board: Name of the board to look for history. Default is None.
@param pool: Name of the pool to look for history. Default is None.
@param print_each_interval: True display all intervals, default is False.
@returns: stats report for this particular host. The report is a list of
tuples (stat_string, intervals, hostname), intervals is a sorted
dictionary.
"""
if hosts:
board=None
pool=None
hosts_intervals = get_intervals_for_hosts(t_start, t_end, hosts, board,
pool)
history = {}
pool = multiprocessing.pool.ThreadPool(processes=16)
args = []
for hostname,intervals in hosts_intervals.items():
args.append({'t_start': t_start,
't_end': t_end,
'hostname': hostname,
'intervals': intervals})
results = pool.imap_unordered(get_host_history_intervals, args)
for hostname, intervals, count in results:
history[hostname] = (intervals, count)
report = []
for hostname,intervals in history.items():
total_times = calculate_total_times(intervals[0])
stats = get_stats_string(
t_start, t_end, total_times, intervals[0], hostname,
intervals[1], print_each_interval)
report.append((stats, intervals[0], hostname))
return report
def get_report_for_host(t_start, t_end, hostname, print_each_interval):
"""Gets stats report for a host
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param hostname: hostname for the host we are interested in (string)
@param print_each_interval: True or False, whether we want to
display all intervals
@returns: stats report for this particular host (string)
"""
# Search for status change intervals during given time range.
intervals = get_intervals_for_host(t_start, t_end, hostname)
num_entries_found = len(intervals)
# Update the status change intervals with status before the first entry and
# host's lock history.
_, intervals_of_statuses = get_host_history_intervals(
{'t_start': t_start,
't_end': t_end,
'hostname': hostname,
'intervals': intervals})
total_times = calculate_total_times(intervals_of_statuses)
return (get_stats_string(
t_start, t_end, total_times, intervals_of_statuses,
hostname, num_entries_found, print_each_interval),
intervals_of_statuses)
def get_stats_string(t_start, t_end, total_times, intervals_of_statuses,
hostname, num_entries_found, print_each_interval):
"""Returns string reporting host_history for this host.
@param t_start: beginning of time period we are interested in.
@param t_end: end of time period we are interested in.
@param total_times: dictionary where key=status,
value=(time spent in that status)
@param intervals_of_statuses: dictionary where keys is tuple (ti, tf),
and value is the status along with other metadata.
@param hostname: hostname for the host we are interested in (string)
@param num_entries_found: Number of entries found for the host in es
@param print_each_interval: boolean, whether to print each interval
"""
delta = t_end - t_start
result = 'usage stats for host: %s \n' % (hostname)
result += ' %s - %s \n' % (time_utils.epoch_time_to_date_string(t_start),
time_utils.epoch_time_to_date_string(t_end))
result += ' Num entries found in this interval: %s\n' % (num_entries_found)
for status, value in total_times.iteritems():
spaces = (15 - len(status)) * ' '
result += ' %s: %s %.2f %%\n' % (status, spaces, 100*value/delta)
result += '- -- --- ---- ----- ---- --- -- -\n'
if print_each_interval:
for interval, status_info in intervals_of_statuses.iteritems():
t0, t1 = interval
t0_string = time_utils.epoch_time_to_date_string(t0)
t1_string = time_utils.epoch_time_to_date_string(t1)
status = status_info['status']
delta = int(t1-t0)
id_info = status_info['metadata'].get(
'task_id', status_info['metadata'].get('job_id', ''))
result += (' %s : %s %-15s %-10s %ss\n' %
(t0_string, t1_string, status, id_info, delta))
return result
def calculate_status_times(t_start, t_end, int_status, metadata,
locked_intervals):
"""Returns a list of intervals along w/ statuses associated with them.
If the dut is in status Ready, i.e., int_status==Ready, the lock history
should be applied so that the time period when dut is locked is considered
as not available. Any other status is considered that dut is doing something
and being used. `Repair Failed` and Repairing are not checked with lock
status, since these two statuses indicate the dut is not available any way.
@param t_start: start time
@param t_end: end time
@param int_status: status of [t_start, t_end] if not locked
@param metadata: metadata of the status change, e.g., task_id, task_name.
@param locked_intervals: list of tuples denoting intervals of locked states
@returns: dictionary where key = (t_interval_start, t_interval_end),
val = (status, metadata)
t_interval_start: beginning of interval for that status
t_interval_end: end of the interval for that status
status: string such as 'Repair Failed', 'Locked', etc.
metadata: A dictionary of metadata, e.g.,
{'task_id':123, 'task_name':'Reset'}
"""
statuses = collections.OrderedDict()
prev_interval_end = t_start
# TODO: Put allow more information here in info/locked status
status_info = {'status': int_status,
'metadata': metadata}
locked_info = {'status': 'Locked',
'metadata': {}}
if not locked_intervals:
statuses[(t_start, t_end)] = status_info
return statuses
for lock_start, lock_end in locked_intervals:
if prev_interval_end >= t_end:
break
if lock_start > t_end:
# optimization to break early
# case 0
# Timeline of status change: t_start t_end
# Timeline of lock action: lock_start lock_end
break
elif lock_end < prev_interval_end:
# case 1
# prev_interval_end t_end
# lock_start lock_end
continue
elif lock_end <= t_end and lock_start >= prev_interval_end:
# case 2
# prev_interval_end t_end
# lock_start lock_end
# Lock happened in the middle, while the host stays in the same
# status, consider the lock has no effect on host history.
statuses[(prev_interval_end, lock_end)] = status_info
prev_interval_end = lock_end
elif lock_end > prev_interval_end and lock_start < prev_interval_end:
# case 3
# prev_interval_end t_end
# lock_start lock_end (or lock_end)
# If the host status changed in the middle of being locked, consider
# the new status change as part of the host history.
statuses[(prev_interval_end, min(lock_end, t_end))] = locked_info
prev_interval_end = lock_end
elif lock_start < t_end and lock_end > t_end:
# case 4
# prev_interval_end t_end
# lock_start lock_end
# If the lock happens in the middle of host status change, consider
# the lock has no effect on the host history for that status.
statuses[(prev_interval_end, t_end)] = status_info
statuses[(lock_start, t_end)] = locked_info
prev_interval_end = t_end
# Otherwise we are in the case where lock_end < t_start OR
# lock_start > t_end, which means the lock doesn't apply.
if t_end > prev_interval_end:
# This is to avoid logging the same time
statuses[(prev_interval_end, t_end)] = status_info
return statuses
def get_log_url(hostname, metadata):
"""Compile a url to job's debug log from debug string.
@param hostname: Hostname of the dut.
@param metadata: A dictionary of other metadata, e.g.,
{'task_id':123, 'task_name':'Reset'}
@return: Url of the debug log for special task or job url for test job.
"""
log_url = None
if 'task_id' in metadata and 'task_name' in metadata:
log_url = job_history.TASK_URL % {'hostname': hostname,
'task_id': metadata['task_id'],
'task_name': metadata['task_name']}
elif 'job_id' in metadata and 'owner' in metadata:
log_url = job_history.JOB_URL % {'hostname': hostname,
'job_id': metadata['job_id'],
'owner': metadata['owner']}
return log_url
def build_history(hostname, status_intervals):
"""Get host history information from given state intervals.
@param hostname: Hostname of the dut.
@param status_intervals: A ordered dictionary with
key as (t_start, t_end) and value as (status, metadata)
status = status of the host. e.g. 'Repair Failed'
t_start is the beginning of the interval where the DUT's has
that status
t_end is the end of the interval where the DUT has that
status
metadata: A dictionary of other metadata, e.g.,
{'task_id':123, 'task_name':'Reset'}
@return: A list of host history, e.g.,
[{'status': 'Resetting'
'start_time': '2014-08-07 10:02:16',
'end_time': '2014-08-07 10:03:16',
'log_url': 'http://autotest/reset-546546/debug',
'task_id': 546546},
{'status': 'Running'
'start_time': '2014-08-07 10:03:18',
'end_time': '2014-08-07 10:13:00',
'log_url': 'http://autotest/afe/#tab_id=view_job&object_id=1683',
'job_id': 1683}
]
"""
history = []
for time_interval, status_info in status_intervals.items():
start_time = time_utils.epoch_time_to_date_string(time_interval[0])
end_time = time_utils.epoch_time_to_date_string(time_interval[1])
interval = {'status': status_info['status'],
'start_time': start_time,
'end_time': end_time}
interval['log_url'] = get_log_url(hostname, status_info['metadata'])
interval.update(status_info['metadata'])
history.append(interval)
return history
def get_status_intervals(history_details):
"""Get a list of status interval from history details.
This is a reverse method of above build_history. Caller gets the history
details from RPC get_host_history, and use this method to get the list of
status interval, which can be used to calculate stats from
host_history_utils.aggregate_hosts.
@param history_details: A dictionary of host history for each host, e.g.,
{'172.22.33.51': [{'status': 'Resetting'
'start_time': '2014-08-07 10:02:16',
'end_time': '2014-08-07 10:03:16',
'log_url': 'http://autotest/reset-546546/debug',
'task_id': 546546},]
}
@return: A list of dictionaries where keys are tuple (start_time, end_time),
and value is a dictionary containing at least key 'status'.
"""
status_intervals = []
for host,history in history_details.iteritems():
intervals = collections.OrderedDict()
for interval in history:
start_time = time_utils.to_epoch_time(interval['start_time'])
end_time = time_utils.to_epoch_time(interval['end_time'])
metadata = copy.deepcopy(interval)
metadata['hostname'] = host
intervals[(start_time, end_time)] = {'status': interval['status'],
'metadata': metadata}
status_intervals.append(intervals)
return status_intervals
def get_machine_utilization_rate(stats):
"""Get machine utilization rate from given stats.
@param stats: A dictionary with a status as key and value is the total
number of seconds spent on the status.
@return: The percentage of time when dut is running test jobs.
"""
not_utilized_status = ['Repairing', 'Repair Failed', 'Ready', 'Verifying']
excluded_status = ['Locked']
total_time = 0
total_time_not_utilized = 0.0
for status, interval in stats.iteritems():
if status in excluded_status:
continue
total_time += interval
if status in not_utilized_status:
total_time_not_utilized += interval
if total_time == 0:
# All duts are locked, assume MUR is 0%
return 0
else:
return 1 - total_time_not_utilized/total_time
def get_machine_availability_rate(stats):
"""Get machine availability rate from given stats.
@param stats: A dictionary with a status as key and value is the total
number of seconds spent on the status.
@return: The percentage of time when dut is available to run jobs.
"""
not_available_status = ['Repairing', 'Repair Failed', 'Verifying']
excluded_status = ['Locked']
total_time = 0
total_time_not_available = 0.0
for status, interval in stats.iteritems():
if status in excluded_status:
continue
total_time += interval
if status in not_available_status:
total_time_not_available += interval
if total_time == 0:
# All duts are locked, assume MAR is 0%
return 0
else:
return 1 - total_time_not_available/total_time