# pylint: disable-msg=C0111 import base64, os, tempfile, pickle, datetime, django.db import os.path, getpass from math import sqrt # When you import matplotlib, it tries to write some temp files for better # performance, and it does that to the directory in MPLCONFIGDIR, or, if that # doesn't exist, the home directory. Problem is, the home directory is not # writable when running under Apache, and matplotlib's not smart enough to # handle that. It does appear smart enough to handle the files going # away after they are written, though. temp_dir = os.path.join(tempfile.gettempdir(), '.matplotlib-%s' % getpass.getuser()) if not os.path.exists(temp_dir): os.mkdir(temp_dir) os.environ['MPLCONFIGDIR'] = temp_dir try: import matplotlib matplotlib.use('Agg') import matplotlib.figure, matplotlib.backends.backend_agg import StringIO, colorsys, PIL.Image, PIL.ImageChops except ImportError: # Do nothing, in case this is part of a unit test, so the unit test # can proceed. pass from autotest_lib.frontend.afe import readonly_connection from autotest_lib.frontend.afe.model_logic import ValidationError from json import encoder from autotest_lib.client.common_lib import global_config from autotest_lib.frontend.tko import models, tko_rpc_utils _FIGURE_DPI = 100 _FIGURE_WIDTH_IN = 10 _FIGURE_BOTTOM_PADDING_IN = 2 # for x-axis labels _SINGLE_PLOT_HEIGHT = 6 _MULTIPLE_PLOT_HEIGHT_PER_PLOT = 4 _MULTIPLE_PLOT_MARKER_TYPE = 'o' _MULTIPLE_PLOT_MARKER_SIZE = 4 _SINGLE_PLOT_STYLE = 'bs-' # blue squares with lines connecting _SINGLE_PLOT_ERROR_BAR_COLOR = 'r' _LEGEND_FONT_SIZE = 'xx-small' _LEGEND_HANDLE_LENGTH = 0.03 _LEGEND_NUM_POINTS = 3 _LEGEND_MARKER_TYPE = 'o' _LINE_XTICK_LABELS_SIZE = 'x-small' _BAR_XTICK_LABELS_SIZE = 8 _json_encoder = encoder.JSONEncoder() class NoDataError(Exception): """\ Exception to raise if the graphing query returned an empty resultset. """ def _colors(n): """\ Generator function for creating n colors. The return value is a tuple representing the RGB of the color. """ for i in xrange(n): yield colorsys.hsv_to_rgb(float(i) / n, 1.0, 1.0) def _resort(kernel_labels, list_to_sort): """\ Resorts a list, using a list of kernel strings as the keys. Returns the resorted list. """ labels = [tko_rpc_utils.KernelString(label) for label in kernel_labels] resorted_pairs = sorted(zip(labels, list_to_sort)) # We only want the resorted list; we are not interested in the kernel # strings. return [pair[1] for pair in resorted_pairs] def _quote(string): return "%s%s%s" % ("'", string.replace("'", r"\'"), "'") _HTML_TEMPLATE = """\ <html><head></head><body> <img src="data:image/png;base64,%s" usemap="#%s" border="0" alt="graph"> <map name="%s">%s</map> </body></html>""" _AREA_TEMPLATE = """\ <area shape="rect" coords="%i,%i,%i,%i" title="%s" href="#" onclick="%s(%s); return false;">""" class MetricsPlot(object): def __init__(self, query_dict, plot_type, inverted_series, normalize_to, drilldown_callback): """ query_dict: dictionary containing the main query and the drilldown queries. The main query returns a row for each x value. The first column contains the x-axis label. Subsequent columns contain data for each series, named by the column names. A column named 'errors-<x>' will be interpreted as errors for the series named <x>. plot_type: 'Line' or 'Bar', depending on the plot type the user wants inverted_series: list of series that should be plotted on an inverted y-axis normalize_to: None - do not normalize 'first' - normalize against the first data point 'x__%s' - normalize against the x-axis value %s 'series__%s' - normalize against the series %s drilldown_callback: name of drilldown callback method. """ self.query_dict = query_dict if plot_type == 'Line': self.is_line = True elif plot_type == 'Bar': self.is_line = False else: raise ValidationError({'plot' : 'Plot must be either Line or Bar'}) self.plot_type = plot_type self.inverted_series = inverted_series self.normalize_to = normalize_to if self.normalize_to is None: self.normalize_to = '' self.drilldown_callback = drilldown_callback class QualificationHistogram(object): def __init__(self, query, filter_string, interval, drilldown_callback): """ query: the main query to retrieve the pass rate information. The first column contains the hostnames of all the machines that satisfied the global filter. The second column (titled 'total') contains the total number of tests that ran on that machine and satisfied the global filter. The third column (titled 'good') contains the number of those tests that passed on that machine. filter_string: filter to apply to the common global filter to show the Table View drilldown of a histogram bucket interval: interval for each bucket. E.g., 10 means that buckets should be 0-10%, 10%-20%, ... """ self.query = query self.filter_string = filter_string self.interval = interval self.drilldown_callback = drilldown_callback def _create_figure(height_inches): """\ Creates an instance of matplotlib.figure.Figure, given the height in inches. Returns the figure and the height in pixels. """ fig = matplotlib.figure.Figure( figsize=(_FIGURE_WIDTH_IN, height_inches + _FIGURE_BOTTOM_PADDING_IN), dpi=_FIGURE_DPI, facecolor='white') fig.subplots_adjust(bottom=float(_FIGURE_BOTTOM_PADDING_IN) / height_inches) return (fig, fig.get_figheight() * _FIGURE_DPI) def _create_line(plots, labels, plot_info): """\ Given all the data for the metrics, create a line plot. plots: list of dicts containing the plot data. Each dict contains: x: list of x-values for the plot y: list of corresponding y-values errors: errors for each data point, or None if no error information available label: plot title labels: list of x-tick labels plot_info: a MetricsPlot """ # when we're doing any kind of normalization, all series get put into a # single plot single = bool(plot_info.normalize_to) area_data = [] lines = [] if single: plot_height = _SINGLE_PLOT_HEIGHT else: plot_height = _MULTIPLE_PLOT_HEIGHT_PER_PLOT * len(plots) figure, height = _create_figure(plot_height) if single: subplot = figure.add_subplot(1, 1, 1) # Plot all the data for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))): needs_invert = (plot['label'] in plot_info.inverted_series) # Add a new subplot, if user wants multiple subplots # Also handle axis inversion for subplots here if not single: subplot = figure.add_subplot(len(plots), 1, plot_index + 1) subplot.set_title(plot['label']) if needs_invert: # for separate plots, just invert the y-axis subplot.set_ylim(1, 0) elif needs_invert: # for a shared plot (normalized data), need to invert the y values # manually, since all plots share a y-axis plot['y'] = [-y for y in plot['y']] # Plot the series subplot.set_xticks(range(0, len(labels))) subplot.set_xlim(-1, len(labels)) if single: lines += subplot.plot(plot['x'], plot['y'], label=plot['label'], marker=_MULTIPLE_PLOT_MARKER_TYPE, markersize=_MULTIPLE_PLOT_MARKER_SIZE) error_bar_color = lines[-1].get_color() else: lines += subplot.plot(plot['x'], plot['y'], _SINGLE_PLOT_STYLE, label=plot['label']) error_bar_color = _SINGLE_PLOT_ERROR_BAR_COLOR if plot['errors']: subplot.errorbar(plot['x'], plot['y'], linestyle='None', yerr=plot['errors'], color=error_bar_color) subplot.set_xticklabels([]) # Construct the information for the drilldowns. # We need to do this in a separate loop so that all the data is in # matplotlib before we start calling transform(); otherwise, it will return # incorrect data because it hasn't finished adjusting axis limits. for line in lines: # Get the pixel coordinates of each point on the figure x = line.get_xdata() y = line.get_ydata() label = line.get_label() icoords = line.get_transform().transform(zip(x,y)) # Get the appropriate drilldown query drill = plot_info.query_dict['__' + label + '__'] # Set the title attributes (hover-over tool-tips) x_labels = [labels[x_val] for x_val in x] titles = ['%s - %s: %f' % (label, x_label, y_val) for x_label, y_val in zip(x_labels, y)] # Get the appropriate parameters for the drilldown query params = [dict(query=drill, series=line.get_label(), param=x_label) for x_label in x_labels] area_data += [dict(left=ix - 5, top=height - iy - 5, right=ix + 5, bottom=height - iy + 5, title= title, callback=plot_info.drilldown_callback, callback_arguments=param_dict) for (ix, iy), title, param_dict in zip(icoords, titles, params)] subplot.set_xticklabels(labels, rotation=90, size=_LINE_XTICK_LABELS_SIZE) # Show the legend if there are not multiple subplots if single: font_properties = matplotlib.font_manager.FontProperties( size=_LEGEND_FONT_SIZE) legend = figure.legend(lines, [plot['label'] for plot in plots], prop=font_properties, handlelen=_LEGEND_HANDLE_LENGTH, numpoints=_LEGEND_NUM_POINTS) # Workaround for matplotlib not keeping all line markers in the legend - # it seems if we don't do this, matplotlib won't keep all the line # markers in the legend. for line in legend.get_lines(): line.set_marker(_LEGEND_MARKER_TYPE) return (figure, area_data) def _get_adjusted_bar(x, bar_width, series_index, num_plots): """\ Adjust the list 'x' to take the multiple series into account. Each series should be shifted such that the middle series lies at the appropriate x-axis tick with the other bars around it. For example, if we had four series (i.e. four bars per x value), we want to shift the left edges of the bars as such: Bar 1: -2 * width Bar 2: -width Bar 3: none Bar 4: width """ adjust = (-0.5 * num_plots - 1 + series_index) * bar_width return [x_val + adjust for x_val in x] # TODO(showard): merge much of this function with _create_line by extracting and # parameterizing methods def _create_bar(plots, labels, plot_info): """\ Given all the data for the metrics, create a line plot. plots: list of dicts containing the plot data. x: list of x-values for the plot y: list of corresponding y-values errors: errors for each data point, or None if no error information available label: plot title labels: list of x-tick labels plot_info: a MetricsPlot """ area_data = [] bars = [] figure, height = _create_figure(_SINGLE_PLOT_HEIGHT) # Set up the plot subplot = figure.add_subplot(1, 1, 1) subplot.set_xticks(range(0, len(labels))) subplot.set_xlim(-1, len(labels)) subplot.set_xticklabels(labels, rotation=90, size=_BAR_XTICK_LABELS_SIZE) # draw a bold line at y=0, making it easier to tell if bars are dipping # below the axis or not. subplot.axhline(linewidth=2, color='black') # width here is the width for each bar in the plot. Matplotlib default is # 0.8. width = 0.8 / len(plots) # Plot the data for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))): # Invert the y-axis if needed if plot['label'] in plot_info.inverted_series: plot['y'] = [-y for y in plot['y']] adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1, len(plots)) bar_data = subplot.bar(adjusted_x, plot['y'], width=width, yerr=plot['errors'], facecolor=color, label=plot['label']) bars.append(bar_data[0]) # Construct the information for the drilldowns. # See comment in _create_line for why we need a separate loop to do this. for plot_index, plot in enumerate(plots): adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1, len(plots)) # Let matplotlib plot the data, so that we can get the data-to-image # coordinate transforms line = subplot.plot(adjusted_x, plot['y'], linestyle='None')[0] label = plot['label'] upper_left_coords = line.get_transform().transform(zip(adjusted_x, plot['y'])) bottom_right_coords = line.get_transform().transform( [(x + width, 0) for x in adjusted_x]) # Get the drilldown query drill = plot_info.query_dict['__' + label + '__'] # Set the title attributes x_labels = [labels[x] for x in plot['x']] titles = ['%s - %s: %f' % (plot['label'], label, y) for label, y in zip(x_labels, plot['y'])] params = [dict(query=drill, series=plot['label'], param=x_label) for x_label in x_labels] area_data += [dict(left=ulx, top=height - uly, right=brx, bottom=height - bry, title=title, callback=plot_info.drilldown_callback, callback_arguments=param_dict) for (ulx, uly), (brx, bry), title, param_dict in zip(upper_left_coords, bottom_right_coords, titles, params)] figure.legend(bars, [plot['label'] for plot in plots]) return (figure, area_data) def _normalize(data_values, data_errors, base_values, base_errors): """\ Normalize the data against a baseline. data_values: y-values for the to-be-normalized data data_errors: standard deviations for the to-be-normalized data base_values: list of values normalize against base_errors: list of standard deviations for those base values """ values = [] for value, base in zip(data_values, base_values): try: values.append(100 * (value - base) / base) except ZeroDivisionError: # Base is 0.0 so just simplify: # If value < base: append -100.0; # If value == base: append 0.0 (obvious); and # If value > base: append 100.0. values.append(100 * float(cmp(value, base))) # Based on error for f(x,y) = 100 * (x - y) / y if data_errors: if not base_errors: base_errors = [0] * len(data_errors) errors = [] for data, error, base_value, base_error in zip( data_values, data_errors, base_values, base_errors): try: errors.append(sqrt(error**2 * (100 / base_value)**2 + base_error**2 * (100 * data / base_value**2)**2 + error * base_error * (100 / base_value**2)**2)) except ZeroDivisionError: # Again, base is 0.0 so do the simple thing. errors.append(100 * abs(error)) else: errors = None return (values, errors) def _create_png(figure): """\ Given the matplotlib figure, generate the PNG data for it. """ # Draw the image canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(figure) canvas.draw() size = canvas.get_renderer().get_canvas_width_height() image_as_string = canvas.tostring_rgb() image = PIL.Image.fromstring('RGB', size, image_as_string, 'raw', 'RGB', 0, 1) image_background = PIL.Image.new(image.mode, image.size, figure.get_facecolor()) # Crop the image to remove surrounding whitespace non_whitespace = PIL.ImageChops.difference(image, image_background) bounding_box = non_whitespace.getbbox() image = image.crop(bounding_box) image_data = StringIO.StringIO() image.save(image_data, format='PNG') return image_data.getvalue(), bounding_box def _create_image_html(figure, area_data, plot_info): """\ Given the figure and drilldown data, construct the HTML that will render the graph as a PNG image, and attach the image map to that image. figure: figure containing the drawn plot(s) area_data: list of parameters for each area of the image map. See the definition of the template string '_AREA_TEMPLATE' plot_info: a MetricsPlot or QualHistogram """ png, bbox = _create_png(figure) # Construct the list of image map areas areas = [_AREA_TEMPLATE % (data['left'] - bbox[0], data['top'] - bbox[1], data['right'] - bbox[0], data['bottom'] - bbox[1], data['title'], data['callback'], _json_encoder.encode(data['callback_arguments']) .replace('"', '"')) for data in area_data] map_name = plot_info.drilldown_callback + '_map' return _HTML_TEMPLATE % (base64.b64encode(png), map_name, map_name, '\n'.join(areas)) def _find_plot_by_label(plots, label): for index, plot in enumerate(plots): if plot['label'] == label: return index raise ValueError('no plot labeled "%s" found' % label) def _normalize_to_series(plots, base_series): base_series_index = _find_plot_by_label(plots, base_series) base_plot = plots[base_series_index] base_xs = base_plot['x'] base_values = base_plot['y'] base_errors = base_plot['errors'] del plots[base_series_index] for plot in plots: old_xs, old_values, old_errors = plot['x'], plot['y'], plot['errors'] new_xs, new_values, new_errors = [], [], [] new_base_values, new_base_errors = [], [] # Select only points in the to-be-normalized data that have a # corresponding baseline value for index, x_value in enumerate(old_xs): try: base_index = base_xs.index(x_value) except ValueError: continue new_xs.append(x_value) new_values.append(old_values[index]) new_base_values.append(base_values[base_index]) if old_errors: new_errors.append(old_errors[index]) new_base_errors.append(base_errors[base_index]) if not new_xs: raise NoDataError('No normalizable data for series ' + plot['label']) plot['x'] = new_xs plot['y'] = new_values if old_errors: plot['errors'] = new_errors plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'], new_base_values, new_base_errors) def _create_metrics_plot_helper(plot_info, extra_text=None): """ Create a metrics plot of the given plot data. plot_info: a MetricsPlot object. extra_text: text to show at the uppper-left of the graph TODO(showard): move some/all of this logic into methods on MetricsPlot """ query = plot_info.query_dict['__main__'] cursor = readonly_connection.cursor() cursor.execute(query) if not cursor.rowcount: raise NoDataError('query did not return any data') rows = cursor.fetchall() # "transpose" rows, so columns[0] is all the values from the first column, # etc. columns = zip(*rows) plots = [] labels = [str(label) for label in columns[0]] needs_resort = (cursor.description[0][0] == 'kernel') # Collect all the data for the plot col = 1 while col < len(cursor.description): y = columns[col] label = cursor.description[col][0] col += 1 if (col < len(cursor.description) and 'errors-' + label == cursor.description[col][0]): errors = columns[col] col += 1 else: errors = None if needs_resort: y = _resort(labels, y) if errors: errors = _resort(labels, errors) x = [index for index, value in enumerate(y) if value is not None] if not x: raise NoDataError('No data for series ' + label) y = [y[i] for i in x] if errors: errors = [errors[i] for i in x] plots.append({ 'label': label, 'x': x, 'y': y, 'errors': errors }) if needs_resort: labels = _resort(labels, labels) # Normalize the data if necessary normalize_to = plot_info.normalize_to if normalize_to == 'first' or normalize_to.startswith('x__'): if normalize_to != 'first': baseline = normalize_to[3:] try: baseline_index = labels.index(baseline) except ValueError: raise ValidationError({ 'Normalize' : 'Invalid baseline %s' % baseline }) for plot in plots: if normalize_to == 'first': plot_index = 0 else: try: plot_index = plot['x'].index(baseline_index) # if the value is not found, then we cannot normalize except ValueError: raise ValidationError({ 'Normalize' : ('%s does not have a value for %s' % (plot['label'], normalize_to[3:])) }) base_values = [plot['y'][plot_index]] * len(plot['y']) if plot['errors']: base_errors = [plot['errors'][plot_index]] * len(plot['errors']) plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'], base_values, None or base_errors) elif normalize_to.startswith('series__'): base_series = normalize_to[8:] _normalize_to_series(plots, base_series) # Call the appropriate function to draw the line or bar plot if plot_info.is_line: figure, area_data = _create_line(plots, labels, plot_info) else: figure, area_data = _create_bar(plots, labels, plot_info) # TODO(showard): extract these magic numbers to named constants if extra_text: text_y = .95 - .0075 * len(plots) figure.text(.1, text_y, extra_text, size='xx-small') return (figure, area_data) def create_metrics_plot(query_dict, plot_type, inverted_series, normalize_to, drilldown_callback, extra_text=None): plot_info = MetricsPlot(query_dict, plot_type, inverted_series, normalize_to, drilldown_callback) figure, area_data = _create_metrics_plot_helper(plot_info, extra_text) return _create_image_html(figure, area_data, plot_info) def _get_hostnames_in_bucket(hist_data, bucket): """\ Get all the hostnames that constitute a particular bucket in the histogram. hist_data: list containing tuples of (hostname, pass_rate) bucket: tuple containing the (low, high) values of the target bucket """ return [hostname for hostname, pass_rate in hist_data if bucket[0] <= pass_rate < bucket[1]] def _create_qual_histogram_helper(plot_info, extra_text=None): """\ Create a machine qualification histogram of the given data. plot_info: a QualificationHistogram extra_text: text to show at the upper-left of the graph TODO(showard): move much or all of this into methods on QualificationHistogram """ cursor = readonly_connection.cursor() cursor.execute(plot_info.query) if not cursor.rowcount: raise NoDataError('query did not return any data') # Lists to store the plot data. # hist_data store tuples of (hostname, pass_rate) for machines that have # pass rates between 0 and 100%, exclusive. # no_tests is a list of machines that have run none of the selected tests # no_pass is a list of machines with 0% pass rate # perfect is a list of machines with a 100% pass rate hist_data = [] no_tests = [] no_pass = [] perfect = [] # Construct the lists of data to plot for hostname, total, good in cursor.fetchall(): if total == 0: no_tests.append(hostname) continue if good == 0: no_pass.append(hostname) elif good == total: perfect.append(hostname) else: percentage = 100.0 * good / total hist_data.append((hostname, percentage)) interval = plot_info.interval bins = range(0, 100, interval) if bins[-1] != 100: bins.append(bins[-1] + interval) figure, height = _create_figure(_SINGLE_PLOT_HEIGHT) subplot = figure.add_subplot(1, 1, 1) # Plot the data and get all the bars plotted _,_, bars = subplot.hist([data[1] for data in hist_data], bins=bins, align='left') bars += subplot.bar([-interval], len(no_pass), width=interval, align='center') bars += subplot.bar([bins[-1]], len(perfect), width=interval, align='center') bars += subplot.bar([-3 * interval], len(no_tests), width=interval, align='center') buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]] # set the x-axis range to cover all the normal bins plus the three "special" # ones - N/A (3 intervals left), 0% (1 interval left) ,and 100% (far right) subplot.set_xlim(-4 * interval, bins[-1] + interval) subplot.set_xticks([-3 * interval, -interval] + bins + [100 + interval]) subplot.set_xticklabels(['N/A', '0%'] + ['%d%% - <%d%%' % bucket for bucket in buckets] + ['100%'], rotation=90, size='small') # Find the coordinates on the image for each bar x = [] y = [] for bar in bars: x.append(bar.get_x()) y.append(bar.get_height()) f = subplot.plot(x, y, linestyle='None')[0] upper_left_coords = f.get_transform().transform(zip(x, y)) bottom_right_coords = f.get_transform().transform( [(x_val + interval, 0) for x_val in x]) # Set the title attributes titles = ['%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val) for bucket, y_val in zip(buckets, y)] titles.append('0%%: %d machines' % len(no_pass)) titles.append('100%%: %d machines' % len(perfect)) titles.append('N/A: %d machines' % len(no_tests)) # Get the hostnames for each bucket in the histogram names_list = [_get_hostnames_in_bucket(hist_data, bucket) for bucket in buckets] names_list += [no_pass, perfect] if plot_info.filter_string: plot_info.filter_string += ' AND ' # Construct the list of drilldown parameters to be passed when the user # clicks on the bar. params = [] for names in names_list: if names: hostnames = ','.join(_quote(hostname) for hostname in names) hostname_filter = 'hostname IN (%s)' % hostnames full_filter = plot_info.filter_string + hostname_filter params.append({'type': 'normal', 'filterString': full_filter}) else: params.append({'type': 'empty'}) params.append({'type': 'not_applicable', 'hosts': '<br />'.join(no_tests)}) area_data = [dict(left=ulx, top=height - uly, right=brx, bottom=height - bry, title=title, callback=plot_info.drilldown_callback, callback_arguments=param_dict) for (ulx, uly), (brx, bry), title, param_dict in zip(upper_left_coords, bottom_right_coords, titles, params)] # TODO(showard): extract these magic numbers to named constants if extra_text: figure.text(.1, .95, extra_text, size='xx-small') return (figure, area_data) def create_qual_histogram(query, filter_string, interval, drilldown_callback, extra_text=None): plot_info = QualificationHistogram(query, filter_string, interval, drilldown_callback) figure, area_data = _create_qual_histogram_helper(plot_info, extra_text) return _create_image_html(figure, area_data, plot_info) def create_embedded_plot(model, update_time): """\ Given an EmbeddedGraphingQuery object, generate the PNG image for it. model: EmbeddedGraphingQuery object update_time: 'Last updated' time """ params = pickle.loads(model.params) extra_text = 'Last updated: %s' % update_time if model.graph_type == 'metrics': plot_info = MetricsPlot(query_dict=params['queries'], plot_type=params['plot'], inverted_series=params['invert'], normalize_to=None, drilldown_callback='') figure, areas_unused = _create_metrics_plot_helper(plot_info, extra_text) elif model.graph_type == 'qual': plot_info = QualificationHistogram( query=params['query'], filter_string=params['filter_string'], interval=params['interval'], drilldown_callback='') figure, areas_unused = _create_qual_histogram_helper(plot_info, extra_text) else: raise ValueError('Invalid graph_type %s' % model.graph_type) image, bounding_box_unused = _create_png(figure) return image _cache_timeout = global_config.global_config.get_config_value( 'AUTOTEST_WEB', 'graph_cache_creation_timeout_minutes') def handle_plot_request(id, max_age): """\ Given the embedding id of a graph, generate a PNG of the embedded graph associated with that id. id: id of the embedded graph max_age: maximum age, in minutes, that a cached version should be held """ model = models.EmbeddedGraphingQuery.objects.get(id=id) # Check if the cached image needs to be updated now = datetime.datetime.now() update_time = model.last_updated + datetime.timedelta(minutes=int(max_age)) if now > update_time: cursor = django.db.connection.cursor() # We want this query to update the refresh_time only once, even if # multiple threads are running it at the same time. That is, only the # first thread will win the race, and it will be the one to update the # cached image; all other threads will show that they updated 0 rows query = """ UPDATE embedded_graphing_queries SET refresh_time = NOW() WHERE id = %s AND ( refresh_time IS NULL OR refresh_time + INTERVAL %s MINUTE < NOW() ) """ cursor.execute(query, (id, _cache_timeout)) # Only refresh the cached image if we were successful in updating the # refresh time if cursor.rowcount: model.cached_png = create_embedded_plot(model, now.ctime()) model.last_updated = now model.refresh_time = None model.save() return model.cached_png