# Lint as: python3 # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Emit warning messages to html or csv files.""" # Many functions in this module have too many arguments to be refactored. # pylint:disable=too-many-arguments,missing-function-docstring # To emit html page of warning messages: # flags: --byproject, --url, --separator # Old stuff for static html components: # html_script_style: static html scripts and styles # htmlbig: # dump_stats, dump_html_prologue, dump_html_epilogue: # emit_buttons: # dump_fixed # sort_warnings: # emit_stats_by_project: # all_patterns, # findproject, classify_warning # dump_html # # New dynamic HTML page's static JavaScript data: # Some data are copied from Python to JavaScript, to generate HTML elements. # FlagPlatform flags.platform # FlagURL flags.url, used by 'android' # FlagSeparator flags.separator, used by 'android' # SeverityColors: list of colors for all severity levels # SeverityHeaders: list of headers for all severity levels # SeverityColumnHeaders: list of column_headers for all severity levels # ProjectNames: project_names, or project_list[*][0] # WarnPatternsSeverity: warn_patterns[*]['severity'] # WarnPatternsDescription: warn_patterns[*]['description'] # WarningMessages: warning_messages # Warnings: warning_records # StatsHeader: warning count table header row # StatsRows: array of warning count table rows # # New dynamic HTML page's dynamic JavaScript data: # # New dynamic HTML related function to emit data: # escape_string, strip_escape_string, emit_warning_arrays # emit_js_data(): from __future__ import print_function import csv import html import sys # pylint:disable=relative-beyond-top-level from .severity import Severity HTML_HEAD_SCRIPTS = """\ """ def make_writer(output_stream): def writer(text): return output_stream.write(text + '\n') return writer def html_big(param): return '' + param + '' def dump_html_prologue(title, writer, warn_patterns, project_names): writer('\n') writer('' + title + '') writer(HTML_HEAD_SCRIPTS) emit_stats_by_project(writer, warn_patterns, project_names) writer('\n') writer(html_big(title)) writer('

') def dump_html_epilogue(writer): writer('\n\n') def sort_warnings(warn_patterns): for i in warn_patterns: i['members'] = sorted(set(i['members'])) def create_warnings(warn_patterns, project_names): """Creates warnings s.t. warnings[p][s] is as specified in above docs. Args: warn_patterns: list of warning patterns for specified platform project_names: list of project names Returns: 2D warnings array where warnings[p][s] is # of warnings in project name p of severity level s """ warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names} for pattern in warn_patterns: value = pattern['severity'].value for project in pattern['projects']: warnings[project][value] += pattern['projects'][project] return warnings def get_total_by_project(warnings, project_names): """Returns dict, project as key and # warnings for that project as value.""" return { p: sum(warnings[p][s.value] for s in Severity.levels) for p in project_names } def get_total_by_severity(warnings, project_names): """Returns dict, severity as key and # warnings of that severity as value.""" return { s.value: sum(warnings[p][s.value] for p in project_names) for s in Severity.levels } def emit_table_header(total_by_severity): """Returns list of HTML-formatted content for severity stats.""" stats_header = ['Project'] for severity in Severity.levels: if total_by_severity[severity.value]: stats_header.append( '{}'.format( severity.color, severity.column_header)) stats_header.append('TOTAL') return stats_header def emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names): """Returns total project warnings and row of stats for each project. Args: warnings: output of create_warnings(warn_patterns, project_names) total_by_project: output of get_total_by_project(project_names) total_by_severity: output of get_total_by_severity(project_names) project_names: list of project names Returns: total_all_projects, the total number of warnings over all projects stats_rows, a 2d list where each row is [Project Name, , total # warnings for this project] """ total_all_projects = 0 stats_rows = [] for p_name in project_names: if total_by_project[p_name]: one_row = [p_name] for severity in Severity.levels: if total_by_severity[severity.value]: one_row.append(warnings[p_name][severity.value]) one_row.append(total_by_project[p_name]) stats_rows.append(one_row) total_all_projects += total_by_project[p_name] return total_all_projects, stats_rows def emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, total_all_projects, writer): """Emits stats_header and stats_rows as specified above. Args: total_by_severity: output of get_total_by_severity() stats_header: output of emit_table_header() stats_rows: output of emit_row_counts_per_project() total_all_projects: output of emit_row_counts_per_project() writer: writer returned by make_writer(output_stream) """ total_all_severities = 0 one_row = ['TOTAL'] for severity in Severity.levels: if total_by_severity[severity.value]: one_row.append(total_by_severity[severity.value]) total_all_severities += total_by_severity[severity.value] one_row.append(total_all_projects) stats_rows.append(one_row) writer('') def emit_stats_by_project(writer, warn_patterns, project_names): """Dump a google chart table of warnings per project and severity.""" warnings = create_warnings(warn_patterns, project_names) total_by_project = get_total_by_project(warnings, project_names) total_by_severity = get_total_by_severity(warnings, project_names) stats_header = emit_table_header(total_by_severity) total_all_projects, stats_rows = emit_row_counts_per_project( warnings, total_by_project, total_by_severity, project_names) emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows, total_all_projects, writer) def dump_stats(writer, warn_patterns): """Dump some stats about total number of warnings and such.""" known = 0 skipped = 0 unknown = 0 sort_warnings(warn_patterns) for i in warn_patterns: if i['severity'] == Severity.UNMATCHED: unknown += len(i['members']) elif i['severity'] == Severity.SKIP: skipped += len(i['members']) else: known += len(i['members']) writer('Number of classified warnings: ' + str(known) + '
') writer('Number of skipped warnings: ' + str(skipped) + '
') writer('Number of unclassified warnings: ' + str(unknown) + '
') total = unknown + known + skipped extra_msg = '' if total < 1000: extra_msg = ' (low count may indicate incremental build)' writer('Total number of warnings: ' + str(total) + '' + extra_msg) # New base table of warnings, [severity, warn_id, project, warning_message] # Need buttons to show warnings in different grouping options. # (1) Current, group by severity, id for each warning pattern # sort by severity, warn_id, warning_message # (2) Current --byproject, group by severity, # id for each warning pattern + project name # sort by severity, warn_id, project, warning_message # (3) New, group by project + severity, # id for each warning pattern # sort by project, severity, warn_id, warning_message def emit_buttons(writer): """Write the button elements in HTML.""" writer('\n' '\n' '\n' '
') def all_patterns(category): patterns = '' for i in category['patterns']: patterns += i patterns += ' / ' return patterns def dump_fixed(writer, warn_patterns): """Show which warnings no longer occur.""" anchor = 'fixed_warnings' mark = anchor + '_mark' writer('\n

' ' Fixed warnings. ' 'No more occurrences. Please consider turning these into ' 'errors if possible, before they are reintroduced in to the build' ':

') writer('
') fixed_patterns = [] for i in warn_patterns: if not i['members']: fixed_patterns.append(i['description'] + ' (' + all_patterns(i) + ')') fixed_patterns = sorted(fixed_patterns) writer('') writer('
') def write_severity(csvwriter, sev, kind, warn_patterns): """Count warnings of given severity and write CSV entries to writer.""" total = 0 for pattern in warn_patterns: if pattern['severity'] == sev and pattern['members']: num_members = len(pattern['members']) total += num_members warning = kind + ': ' + (pattern['description'] or '?') csvwriter.writerow([num_members, '', warning]) # print number of warnings for each project, ordered by project name projects = sorted(pattern['projects'].keys()) for project in projects: csvwriter.writerow([pattern['projects'][project], project, warning]) csvwriter.writerow([total, '', kind + ' warnings']) return total def dump_csv(csvwriter, warn_patterns): """Dump number of warnings in CSV format to writer.""" sort_warnings(warn_patterns) total = 0 for severity in Severity.levels: total += write_severity( csvwriter, severity, severity.column_header, warn_patterns) csvwriter.writerow([total, '', 'All warnings']) def dump_csv_with_description(csvwriter, warning_records, warning_messages, warn_patterns, project_names): """Outputs all the warning messages by project.""" csv_output = [] for record in warning_records: project_name = project_names[record[1]] pattern = warn_patterns[record[0]] severity = pattern['severity'].header category = pattern['category'] description = pattern['description'] warning = warning_messages[record[2]] csv_output.append([project_name, severity, category, description, warning]) csv_output = sorted(csv_output) for output in csv_output: csvwriter.writerow(output) # Return line with escaped backslash and quotation characters. def escape_string(line): return line.replace('\\', '\\\\').replace('"', '\\"') # Return line without trailing '\n' and escape the quotation characters. def strip_escape_string(line): if not line: return line line = line[:-1] if line[-1] == '\n' else line return escape_string(line) def emit_warning_array(name, writer, warn_patterns): writer('var warning_{} = ['.format(name)) for pattern in warn_patterns: if name == 'severity': writer('{},'.format(pattern[name].value)) else: writer('{},'.format(pattern[name])) writer('];') def emit_warning_arrays(writer, warn_patterns): emit_warning_array('severity', writer, warn_patterns) writer('var warning_description = [') for pattern in warn_patterns: if pattern['members']: writer('"{}",'.format(escape_string(pattern['description']))) else: writer('"",') # no such warning writer('];') SCRIPTS_FOR_WARNING_GROUPS = """ function compareMessages(x1, x2) { // of the same warning type return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1; } function byMessageCount(x1, x2) { return x2[2] - x1[2]; // reversed order } function bySeverityMessageCount(x1, x2) { // orer by severity first if (x1[1] != x2[1]) return x1[1] - x2[1]; return byMessageCount(x1, x2); } const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/; function addURL(line) { // used by Android if (FlagURL == "") return line; if (FlagSeparator == "") { return line.replace(ParseLinePattern, "$1:$2:$3"); } return line.replace(ParseLinePattern, "$1:$2:$3"); } function addURLToLine(line, link) { // used by Chrome let line_split = line.split(":"); let path = line_split.slice(0,3).join(":"); let msg = line_split.slice(3).join(":"); let html_link = `${path}${msg}`; return html_link; } function createArrayOfDictionaries(n) { var result = []; for (var i=0; i" + " " + description + " (" + messages.length + ")"; result += ""; } if (result.length > 0) { return "
" + header + ": " + totalMessages + "
" + result + "
"; } return ""; // empty section } function generateSectionsBySeverity() { var result = ""; var groups = groupWarningsBySeverity(); for (s=0; s

') writer('\n') emit_buttons(writer) # Warning messages are grouped by severities or project names. writer('
') if flags.byproject: writer('') else: writer('') dump_fixed(writer, warn_patterns) dump_html_epilogue(writer) def write_html(flags, project_names, warn_patterns, html_path, warning_messages, warning_links, warning_records, header_str): """Write warnings html file.""" if html_path: with open(html_path, 'w') as outf: dump_html(flags, outf, warning_messages, warning_links, warning_records, header_str, warn_patterns, project_names) def write_out_csv(flags, warn_patterns, warning_messages, warning_links, warning_records, header_str, project_names): """Write warnings csv file.""" if flags.csvpath: with open(flags.csvpath, 'w') as outf: dump_csv(csv.writer(outf, lineterminator='\n'), warn_patterns) if flags.csvwithdescription: with open(flags.csvwithdescription, 'w') as outf: dump_csv_with_description(csv.writer(outf, lineterminator='\n'), warning_records, warning_messages, warn_patterns, project_names) if flags.gencsv: dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns) else: dump_html(flags, sys.stdout, warning_messages, warning_links, warning_records, header_str, warn_patterns, project_names)