949205a661
* This new class definition and patterns are shared between Android and ChromeOS compiler tools. * Suppress hard to fix and false positive linter warnings. Test: warn.py --url=http://cs/android --separator='?l=' build.log > warnings.html Test: warn.py --gencsv build.log > warnings.csv Change-Id: Icb47809100ad30796cb1da82610e989d450194fa
895 lines
30 KiB
Python
Executable file
895 lines
30 KiB
Python
Executable file
# python3
|
|
# Copyright (C) 2019 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Grep warnings messages and output HTML tables or warning counts in CSV.
|
|
|
|
Default is to output warnings in HTML tables grouped by warning severity.
|
|
Use option --byproject to output tables grouped by source file projects.
|
|
Use option --gencsv to output warning counts in CSV format.
|
|
"""
|
|
|
|
# List of important data structures and functions in this script.
|
|
#
|
|
# To parse and keep warning message in the input file:
|
|
# severity: classification of message severity
|
|
# warn_patterns:
|
|
# warn_patterns[w]['category'] tool that issued the warning, not used now
|
|
# warn_patterns[w]['description'] table heading
|
|
# warn_patterns[w]['members'] matched warnings from input
|
|
# warn_patterns[w]['option'] compiler flag to control the warning
|
|
# warn_patterns[w]['patterns'] regular expressions to match warnings
|
|
# warn_patterns[w]['projects'][p] number of warnings of pattern w in p
|
|
# warn_patterns[w]['severity'] severity tuple
|
|
# project_list[p][0] project name
|
|
# project_list[p][1] regular expression to match a project path
|
|
# project_patterns[p] re.compile(project_list[p][1])
|
|
# project_names[p] project_list[p][0]
|
|
# warning_messages array of each warning message, without source url
|
|
# warning_records array of [idx to warn_patterns,
|
|
# idx to project_names,
|
|
# idx to warning_messages]
|
|
# android_root
|
|
# platform_version
|
|
# target_product
|
|
# target_variant
|
|
# parse_input_file
|
|
#
|
|
# To emit html page of warning messages:
|
|
# flags: --byproject, --url, --separator
|
|
# Old stuff for static html components:
|
|
# html_script_style: static html scripts and styles
|
|
# htmlbig:
|
|
# dump_stats, dump_html_prologue, dump_html_epilogue:
|
|
# emit_buttons:
|
|
# dump_fixed
|
|
# sort_warnings:
|
|
# emit_stats_by_project:
|
|
# all_patterns,
|
|
# findproject, classify_warning
|
|
# dump_html
|
|
#
|
|
# New dynamic HTML page's static JavaScript data:
|
|
# Some data are copied from Python to JavaScript, to generate HTML elements.
|
|
# FlagURL args.url
|
|
# FlagSeparator args.separator
|
|
# SeverityColors: list of colors for all severity levels
|
|
# SeverityHeaders: list of headers for all severity levels
|
|
# SeverityColumnHeaders: list of column_headers for all severity levels
|
|
# ProjectNames: project_names, or project_list[*][0]
|
|
# WarnPatternsSeverity: warn_patterns[*]['severity']
|
|
# WarnPatternsDescription: warn_patterns[*]['description']
|
|
# WarnPatternsOption: warn_patterns[*]['option']
|
|
# WarningMessages: warning_messages
|
|
# Warnings: warning_records
|
|
# StatsHeader: warning count table header row
|
|
# StatsRows: array of warning count table rows
|
|
#
|
|
# New dynamic HTML page's dynamic JavaScript data:
|
|
#
|
|
# New dynamic HTML related function to emit data:
|
|
# escape_string, strip_escape_string, emit_warning_arrays
|
|
# emit_js_data():
|
|
|
|
from __future__ import print_function
|
|
import argparse
|
|
import cgi
|
|
import csv
|
|
import io
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
import signal
|
|
import sys
|
|
|
|
# pylint:disable=relative-beyond-top-level
|
|
from . import cpp_warn_patterns
|
|
from . import java_warn_patterns
|
|
from . import make_warn_patterns
|
|
from . import other_warn_patterns
|
|
from . import tidy_warn_patterns
|
|
# pylint:disable=g-importing-member
|
|
from .android_project_list import project_list
|
|
from .severity import Severity
|
|
|
|
parser = argparse.ArgumentParser(description='Convert a build log into HTML')
|
|
parser.add_argument('--csvpath',
|
|
help='Save CSV warning file to the passed absolute path',
|
|
default=None)
|
|
parser.add_argument('--gencsv',
|
|
help='Generate a CSV file with number of various warnings',
|
|
action='store_true',
|
|
default=False)
|
|
parser.add_argument('--byproject',
|
|
help='Separate warnings in HTML output by project names',
|
|
action='store_true',
|
|
default=False)
|
|
parser.add_argument('--url',
|
|
help='Root URL of an Android source code tree prefixed '
|
|
'before files in warnings')
|
|
parser.add_argument('--separator',
|
|
help='Separator between the end of a URL and the line '
|
|
'number argument. e.g. #')
|
|
parser.add_argument('--processes',
|
|
type=int,
|
|
default=multiprocessing.cpu_count(),
|
|
help='Number of parallel processes to process warnings')
|
|
parser.add_argument(dest='buildlog', metavar='build.log',
|
|
help='Path to build.log file')
|
|
args = parser.parse_args()
|
|
|
|
warn_patterns = make_warn_patterns.warn_patterns
|
|
warn_patterns.extend(cpp_warn_patterns.warn_patterns)
|
|
warn_patterns.extend(java_warn_patterns.warn_patterns)
|
|
warn_patterns.extend(tidy_warn_patterns.warn_patterns)
|
|
warn_patterns.extend(other_warn_patterns.warn_patterns)
|
|
|
|
project_patterns = []
|
|
project_names = []
|
|
warning_messages = []
|
|
warning_records = []
|
|
|
|
|
|
def initialize_arrays():
|
|
"""Complete global arrays before they are used."""
|
|
global project_names, project_patterns
|
|
project_names = [p[0] for p in project_list]
|
|
project_patterns = [re.compile(p[1]) for p in project_list]
|
|
for w in warn_patterns:
|
|
w['members'] = []
|
|
if 'option' not in w:
|
|
w['option'] = ''
|
|
# Each warning pattern has a 'projects' dictionary, that
|
|
# maps a project name to number of warnings in that project.
|
|
w['projects'] = {}
|
|
|
|
|
|
initialize_arrays()
|
|
|
|
|
|
android_root = ''
|
|
platform_version = 'unknown'
|
|
target_product = 'unknown'
|
|
target_variant = 'unknown'
|
|
|
|
|
|
##### Data and functions to dump html file. ##################################
|
|
|
|
html_head_scripts = """\
|
|
<script type="text/javascript">
|
|
function expand(id) {
|
|
var e = document.getElementById(id);
|
|
var f = document.getElementById(id + "_mark");
|
|
if (e.style.display == 'block') {
|
|
e.style.display = 'none';
|
|
f.innerHTML = '⊕';
|
|
}
|
|
else {
|
|
e.style.display = 'block';
|
|
f.innerHTML = '⊖';
|
|
}
|
|
};
|
|
function expandCollapse(show) {
|
|
for (var id = 1; ; id++) {
|
|
var e = document.getElementById(id + "");
|
|
var f = document.getElementById(id + "_mark");
|
|
if (!e || !f) break;
|
|
e.style.display = (show ? 'block' : 'none');
|
|
f.innerHTML = (show ? '⊖' : '⊕');
|
|
}
|
|
};
|
|
</script>
|
|
<style type="text/css">
|
|
th,td{border-collapse:collapse; border:1px solid black;}
|
|
.button{color:blue;font-size:110%;font-weight:bolder;}
|
|
.bt{color:black;background-color:transparent;border:none;outline:none;
|
|
font-size:140%;font-weight:bolder;}
|
|
.c0{background-color:#e0e0e0;}
|
|
.c1{background-color:#d0d0d0;}
|
|
.t1{border-collapse:collapse; width:100%; border:1px solid black;}
|
|
</style>
|
|
<script src="https://www.gstatic.com/charts/loader.js"></script>
|
|
"""
|
|
|
|
|
|
def html_big(param):
|
|
return '<font size="+2">' + param + '</font>'
|
|
|
|
|
|
def dump_html_prologue(title):
|
|
print('<html>\n<head>')
|
|
print('<title>' + title + '</title>')
|
|
print(html_head_scripts)
|
|
emit_stats_by_project()
|
|
print('</head>\n<body>')
|
|
print(html_big(title))
|
|
print('<p>')
|
|
|
|
|
|
def dump_html_epilogue():
|
|
print('</body>\n</head>\n</html>')
|
|
|
|
|
|
def sort_warnings():
|
|
for i in warn_patterns:
|
|
i['members'] = sorted(set(i['members']))
|
|
|
|
|
|
def emit_stats_by_project():
|
|
"""Dump a google chart table of warnings per project and severity."""
|
|
# warnings[p][s] is number of warnings in project p of severity s.
|
|
# pylint:disable=g-complex-comprehension
|
|
warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names}
|
|
for i in warn_patterns:
|
|
# pytype: disable=attribute-error
|
|
s = i['severity'].value
|
|
# pytype: enable=attribute-error
|
|
for p in i['projects']:
|
|
warnings[p][s] += i['projects'][p]
|
|
|
|
# total_by_project[p] is number of warnings in project p.
|
|
total_by_project = {
|
|
p: sum(warnings[p][s.value] for s in Severity.levels)
|
|
for p in project_names
|
|
}
|
|
|
|
# total_by_severity[s] is number of warnings of severity s.
|
|
total_by_severity = {
|
|
s.value: sum(warnings[p][s.value] for p in project_names)
|
|
for s in Severity.levels
|
|
}
|
|
|
|
# emit table header
|
|
stats_header = ['Project']
|
|
for s in Severity.levels:
|
|
if total_by_severity[s.value]:
|
|
stats_header.append(
|
|
'<span style=\'background-color:{}\'>{}</span>'.format(
|
|
s.color, s.column_header))
|
|
stats_header.append('TOTAL')
|
|
|
|
# emit a row of warning counts per project, skip no-warning projects
|
|
total_all_projects = 0
|
|
stats_rows = []
|
|
for p in project_names:
|
|
if total_by_project[p]:
|
|
one_row = [p]
|
|
for s in Severity.levels:
|
|
if total_by_severity[s.value]:
|
|
one_row.append(warnings[p][s.value])
|
|
one_row.append(total_by_project[p])
|
|
stats_rows.append(one_row)
|
|
total_all_projects += total_by_project[p]
|
|
|
|
# emit a row of warning counts per severity
|
|
total_all_severities = 0
|
|
one_row = ['<b>TOTAL</b>']
|
|
for s in Severity.levels:
|
|
if total_by_severity[s.value]:
|
|
one_row.append(total_by_severity[s.value])
|
|
total_all_severities += total_by_severity[s.value]
|
|
one_row.append(total_all_projects)
|
|
stats_rows.append(one_row)
|
|
print('<script>')
|
|
emit_const_string_array('StatsHeader', stats_header)
|
|
emit_const_object_array('StatsRows', stats_rows)
|
|
print(draw_table_javascript)
|
|
print('</script>')
|
|
|
|
|
|
def dump_stats():
|
|
"""Dump some stats about total number of warnings and such."""
|
|
known = 0
|
|
skipped = 0
|
|
unknown = 0
|
|
sort_warnings()
|
|
for i in warn_patterns:
|
|
if i['severity'] == Severity.UNMATCHED:
|
|
unknown += len(i['members'])
|
|
elif i['severity'] == Severity.SKIP:
|
|
skipped += len(i['members'])
|
|
else:
|
|
known += len(i['members'])
|
|
print('Number of classified warnings: <b>' + str(known) + '</b><br>')
|
|
print('Number of skipped warnings: <b>' + str(skipped) + '</b><br>')
|
|
print('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>')
|
|
total = unknown + known + skipped
|
|
extra_msg = ''
|
|
if total < 1000:
|
|
extra_msg = ' (low count may indicate incremental build)'
|
|
print('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg)
|
|
|
|
|
|
# New base table of warnings, [severity, warn_id, project, warning_message]
|
|
# Need buttons to show warnings in different grouping options.
|
|
# (1) Current, group by severity, id for each warning pattern
|
|
# sort by severity, warn_id, warning_message
|
|
# (2) Current --byproject, group by severity,
|
|
# id for each warning pattern + project name
|
|
# sort by severity, warn_id, project, warning_message
|
|
# (3) New, group by project + severity,
|
|
# id for each warning pattern
|
|
# sort by project, severity, warn_id, warning_message
|
|
def emit_buttons():
|
|
print('<button class="button" onclick="expandCollapse(1);">'
|
|
'Expand all warnings</button>\n'
|
|
'<button class="button" onclick="expandCollapse(0);">'
|
|
'Collapse all warnings</button>\n'
|
|
'<button class="button" onclick="groupBySeverity();">'
|
|
'Group warnings by severity</button>\n'
|
|
'<button class="button" onclick="groupByProject();">'
|
|
'Group warnings by project</button><br>')
|
|
|
|
|
|
def all_patterns(category):
|
|
patterns = ''
|
|
for i in category['patterns']:
|
|
patterns += i
|
|
patterns += ' / '
|
|
return patterns
|
|
|
|
|
|
def dump_fixed():
|
|
"""Show which warnings no longer occur."""
|
|
anchor = 'fixed_warnings'
|
|
mark = anchor + '_mark'
|
|
print('\n<br><p style="background-color:lightblue"><b>'
|
|
'<button id="' + mark + '" '
|
|
'class="bt" onclick="expand(\'' + anchor + '\');">'
|
|
'⊕</button> Fixed warnings. '
|
|
'No more occurrences. Please consider turning these into '
|
|
'errors if possible, before they are reintroduced in to the build'
|
|
':</b></p>')
|
|
print('<blockquote>')
|
|
fixed_patterns = []
|
|
for i in warn_patterns:
|
|
if not i['members']:
|
|
fixed_patterns.append(i['description'] + ' (' +
|
|
all_patterns(i) + ')')
|
|
if i['option']:
|
|
fixed_patterns.append(' ' + i['option'])
|
|
fixed_patterns = sorted(fixed_patterns)
|
|
print('<div id="' + anchor + '" style="display:none;"><table>')
|
|
cur_row_class = 0
|
|
for text in fixed_patterns:
|
|
cur_row_class = 1 - cur_row_class
|
|
# remove last '\n'
|
|
t = text[:-1] if text[-1] == '\n' else text
|
|
print('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>')
|
|
print('</table></div>')
|
|
print('</blockquote>')
|
|
|
|
|
|
def find_project_index(line):
|
|
for p in range(len(project_patterns)):
|
|
if project_patterns[p].match(line):
|
|
return p
|
|
return -1
|
|
|
|
|
|
def classify_one_warning(line, results):
|
|
"""Classify one warning line."""
|
|
for i in range(len(warn_patterns)):
|
|
w = warn_patterns[i]
|
|
for cpat in w['compiled_patterns']:
|
|
# pytype: disable=attribute-error
|
|
if cpat.match(line):
|
|
p = find_project_index(line)
|
|
results.append([line, i, p])
|
|
return
|
|
else:
|
|
# If we end up here, there was a problem parsing the log
|
|
# probably caused by 'make -j' mixing the output from
|
|
# 2 or more concurrent compiles
|
|
pass
|
|
# pytype: enable=attribute-error
|
|
|
|
|
|
def classify_warnings(lines):
|
|
results = []
|
|
for line in lines:
|
|
classify_one_warning(line, results)
|
|
# After the main work, ignore all other signals to a child process,
|
|
# to avoid bad warning/error messages from the exit clean-up process.
|
|
if args.processes > 1:
|
|
signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM))
|
|
return results
|
|
|
|
|
|
def parallel_classify_warnings(warning_lines, parallel_process):
|
|
"""Classify all warning lines with num_cpu parallel processes."""
|
|
num_cpu = args.processes
|
|
if num_cpu > 1:
|
|
groups = [[] for x in range(num_cpu)]
|
|
i = 0
|
|
for x in warning_lines:
|
|
groups[i].append(x)
|
|
i = (i + 1) % num_cpu
|
|
group_results = parallel_process(num_cpu, classify_warnings, groups)
|
|
else:
|
|
group_results = [classify_warnings(warning_lines)]
|
|
|
|
for result in group_results:
|
|
for line, pattern_idx, project_idx in result:
|
|
pattern = warn_patterns[pattern_idx]
|
|
pattern['members'].append(line)
|
|
message_idx = len(warning_messages)
|
|
warning_messages.append(line)
|
|
warning_records.append([pattern_idx, project_idx, message_idx])
|
|
pname = '???' if project_idx < 0 else project_names[project_idx]
|
|
# Count warnings by project.
|
|
if pname in pattern['projects']:
|
|
pattern['projects'][pname] += 1
|
|
else:
|
|
pattern['projects'][pname] = 1
|
|
|
|
|
|
def find_warn_py_and_android_root(path):
|
|
"""Set and return android_root path if it is found."""
|
|
global android_root
|
|
parts = path.split('/')
|
|
for idx in reversed(range(2, len(parts))):
|
|
root_path = '/'.join(parts[:idx])
|
|
# Android root directory should contain this script.
|
|
if os.path.exists(root_path + '/build/make/tools/warn.py'):
|
|
android_root = root_path
|
|
return True
|
|
return False
|
|
|
|
|
|
def find_android_root():
|
|
"""Guess android_root from common prefix of file paths."""
|
|
# Use the longest common prefix of the absolute file paths
|
|
# of the first 10000 warning messages as the android_root.
|
|
global android_root
|
|
warning_lines = set()
|
|
warning_pattern = re.compile('^/[^ ]*/[^ ]*: warning: .*')
|
|
count = 0
|
|
infile = io.open(args.buildlog, mode='r', encoding='utf-8')
|
|
for line in infile:
|
|
if warning_pattern.match(line):
|
|
warning_lines.add(line)
|
|
count += 1
|
|
if count > 9999:
|
|
break
|
|
# Try to find warn.py and use its location to find
|
|
# the source tree root.
|
|
if count < 100:
|
|
path = os.path.normpath(re.sub(':.*$', '', line))
|
|
if find_warn_py_and_android_root(path):
|
|
return
|
|
# Do not use common prefix of a small number of paths.
|
|
if count > 10:
|
|
# pytype: disable=wrong-arg-types
|
|
root_path = os.path.commonprefix(warning_lines)
|
|
# pytype: enable=wrong-arg-types
|
|
if len(root_path) > 2 and root_path[len(root_path) - 1] == '/':
|
|
android_root = root_path[:-1]
|
|
|
|
|
|
def remove_android_root_prefix(path):
|
|
"""Remove android_root prefix from path if it is found."""
|
|
if path.startswith(android_root):
|
|
return path[1 + len(android_root):]
|
|
else:
|
|
return path
|
|
|
|
|
|
def normalize_path(path):
|
|
"""Normalize file path relative to android_root."""
|
|
# If path is not an absolute path, just normalize it.
|
|
path = os.path.normpath(path)
|
|
# Remove known prefix of root path and normalize the suffix.
|
|
if path[0] == '/' and android_root:
|
|
return remove_android_root_prefix(path)
|
|
return path
|
|
|
|
|
|
def normalize_warning_line(line):
|
|
"""Normalize file path relative to android_root in a warning line."""
|
|
# replace fancy quotes with plain ol' quotes
|
|
line = re.sub(u'[\u2018\u2019]', '\'', line)
|
|
# replace non-ASCII chars to spaces
|
|
line = re.sub(u'[^\x00-\x7f]', ' ', line)
|
|
line = line.strip()
|
|
first_column = line.find(':')
|
|
if first_column > 0:
|
|
return normalize_path(line[:first_column]) + line[first_column:]
|
|
else:
|
|
return line
|
|
|
|
|
|
def parse_input_file(infile):
|
|
"""Parse input file, collect parameters and warning lines."""
|
|
global android_root
|
|
global platform_version
|
|
global target_product
|
|
global target_variant
|
|
line_counter = 0
|
|
|
|
# rustc warning messages have two lines that should be combined:
|
|
# warning: description
|
|
# --> file_path:line_number:column_number
|
|
# Some warning messages have no file name:
|
|
# warning: macro replacement list ... [bugprone-macro-parentheses]
|
|
# Some makefile warning messages have no line number:
|
|
# some/path/file.mk: warning: description
|
|
# C/C++ compiler warning messages have line and column numbers:
|
|
# some/path/file.c:line_number:column_number: warning: description
|
|
warning_pattern = re.compile('(^[^ ]*/[^ ]*: warning: .*)|(^warning: .*)')
|
|
warning_without_file = re.compile('^warning: .*')
|
|
rustc_file_position = re.compile('^[ ]+--> [^ ]*/[^ ]*:[0-9]+:[0-9]+')
|
|
|
|
# Collect all warnings into the warning_lines set.
|
|
warning_lines = set()
|
|
prev_warning = ''
|
|
for line in infile:
|
|
if prev_warning:
|
|
if rustc_file_position.match(line):
|
|
# must be a rustc warning, combine 2 lines into one warning
|
|
line = line.strip().replace('--> ', '') + ': ' + prev_warning
|
|
warning_lines.add(normalize_warning_line(line))
|
|
prev_warning = ''
|
|
continue
|
|
# add prev_warning, and then process the current line
|
|
prev_warning = 'unknown_source_file: ' + prev_warning
|
|
warning_lines.add(normalize_warning_line(prev_warning))
|
|
prev_warning = ''
|
|
if warning_pattern.match(line):
|
|
if warning_without_file.match(line):
|
|
# save this line and combine it with the next line
|
|
prev_warning = line
|
|
else:
|
|
warning_lines.add(normalize_warning_line(line))
|
|
continue
|
|
if line_counter < 100:
|
|
# save a little bit of time by only doing this for the first few lines
|
|
line_counter += 1
|
|
m = re.search('(?<=^PLATFORM_VERSION=).*', line)
|
|
if m is not None:
|
|
platform_version = m.group(0)
|
|
m = re.search('(?<=^TARGET_PRODUCT=).*', line)
|
|
if m is not None:
|
|
target_product = m.group(0)
|
|
m = re.search('(?<=^TARGET_BUILD_VARIANT=).*', line)
|
|
if m is not None:
|
|
target_variant = m.group(0)
|
|
m = re.search('.* TOP=([^ ]*) .*', line)
|
|
if m is not None:
|
|
android_root = m.group(1)
|
|
return warning_lines
|
|
|
|
|
|
# Return s with escaped backslash and quotation characters.
|
|
def escape_string(s):
|
|
# pytype: disable=attribute-error
|
|
return s.replace('\\', '\\\\').replace('"', '\\"')
|
|
# pytype: enable=attribute-error
|
|
|
|
|
|
# Return s without trailing '\n' and escape the quotation characters.
|
|
def strip_escape_string(s):
|
|
if not s:
|
|
return s
|
|
s = s[:-1] if s[-1] == '\n' else s
|
|
return escape_string(s)
|
|
|
|
|
|
def emit_warning_array(name):
|
|
print('var warning_{} = ['.format(name))
|
|
for i in range(len(warn_patterns)):
|
|
print('{},'.format(warn_patterns[i][name]))
|
|
print('];')
|
|
|
|
|
|
def emit_warning_arrays():
|
|
emit_warning_array('severity')
|
|
print('var warning_description = [')
|
|
for i in range(len(warn_patterns)):
|
|
if warn_patterns[i]['members']:
|
|
print('"{}",'.format(escape_string(warn_patterns[i]['description'])))
|
|
else:
|
|
print('"",') # no such warning
|
|
print('];')
|
|
|
|
|
|
scripts_for_warning_groups = """
|
|
function compareMessages(x1, x2) { // of the same warning type
|
|
return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1;
|
|
}
|
|
function byMessageCount(x1, x2) {
|
|
return x2[2] - x1[2]; // reversed order
|
|
}
|
|
function bySeverityMessageCount(x1, x2) {
|
|
// orer by severity first
|
|
if (x1[1] != x2[1])
|
|
return x1[1] - x2[1];
|
|
return byMessageCount(x1, x2);
|
|
}
|
|
const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/;
|
|
function addURL(line) {
|
|
if (FlagURL == "") return line;
|
|
if (FlagSeparator == "") {
|
|
return line.replace(ParseLinePattern,
|
|
"<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3");
|
|
}
|
|
return line.replace(ParseLinePattern,
|
|
"<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator +
|
|
"$2'>$1:$2</a>:$3");
|
|
}
|
|
function createArrayOfDictionaries(n) {
|
|
var result = [];
|
|
for (var i=0; i<n; i++) result.push({});
|
|
return result;
|
|
}
|
|
function groupWarningsBySeverity() {
|
|
// groups is an array of dictionaries,
|
|
// each dictionary maps from warning type to array of warning messages.
|
|
var groups = createArrayOfDictionaries(SeverityColors.length);
|
|
for (var i=0; i<Warnings.length; i++) {
|
|
var w = Warnings[i][0];
|
|
var s = WarnPatternsSeverity[w];
|
|
var k = w.toString();
|
|
if (!(k in groups[s]))
|
|
groups[s][k] = [];
|
|
groups[s][k].push(Warnings[i]);
|
|
}
|
|
return groups;
|
|
}
|
|
function groupWarningsByProject() {
|
|
var groups = createArrayOfDictionaries(ProjectNames.length);
|
|
for (var i=0; i<Warnings.length; i++) {
|
|
var w = Warnings[i][0];
|
|
var p = Warnings[i][1];
|
|
var k = w.toString();
|
|
if (!(k in groups[p]))
|
|
groups[p][k] = [];
|
|
groups[p][k].push(Warnings[i]);
|
|
}
|
|
return groups;
|
|
}
|
|
var GlobalAnchor = 0;
|
|
function createWarningSection(header, color, group) {
|
|
var result = "";
|
|
var groupKeys = [];
|
|
var totalMessages = 0;
|
|
for (var k in group) {
|
|
totalMessages += group[k].length;
|
|
groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]);
|
|
}
|
|
groupKeys.sort(bySeverityMessageCount);
|
|
for (var idx=0; idx<groupKeys.length; idx++) {
|
|
var k = groupKeys[idx][0];
|
|
var messages = group[k];
|
|
var w = parseInt(k);
|
|
var wcolor = SeverityColors[WarnPatternsSeverity[w]];
|
|
var description = WarnPatternsDescription[w];
|
|
if (description.length == 0)
|
|
description = "???";
|
|
GlobalAnchor += 1;
|
|
result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" +
|
|
"<button class='bt' id='" + GlobalAnchor + "_mark" +
|
|
"' onclick='expand(\\"" + GlobalAnchor + "\\");'>" +
|
|
"⊕</button> " +
|
|
description + " (" + messages.length + ")</td></tr></table>";
|
|
result += "<div id='" + GlobalAnchor +
|
|
"' style='display:none;'><table class='t1'>";
|
|
var c = 0;
|
|
messages.sort(compareMessages);
|
|
for (var i=0; i<messages.length; i++) {
|
|
result += "<tr><td class='c" + c + "'>" +
|
|
addURL(WarningMessages[messages[i][2]]) + "</td></tr>";
|
|
c = 1 - c;
|
|
}
|
|
result += "</table></div>";
|
|
}
|
|
if (result.length > 0) {
|
|
return "<br><span style='background-color:" + color + "'><b>" +
|
|
header + ": " + totalMessages +
|
|
"</b></span><blockquote><table class='t1'>" +
|
|
result + "</table></blockquote>";
|
|
|
|
}
|
|
return ""; // empty section
|
|
}
|
|
function generateSectionsBySeverity() {
|
|
var result = "";
|
|
var groups = groupWarningsBySeverity();
|
|
for (s=0; s<SeverityColors.length; s++) {
|
|
result += createWarningSection(SeverityHeaders[s], SeverityColors[s], groups[s]);
|
|
}
|
|
return result;
|
|
}
|
|
function generateSectionsByProject() {
|
|
var result = "";
|
|
var groups = groupWarningsByProject();
|
|
for (i=0; i<groups.length; i++) {
|
|
result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]);
|
|
}
|
|
return result;
|
|
}
|
|
function groupWarnings(generator) {
|
|
GlobalAnchor = 0;
|
|
var e = document.getElementById("warning_groups");
|
|
e.innerHTML = generator();
|
|
}
|
|
function groupBySeverity() {
|
|
groupWarnings(generateSectionsBySeverity);
|
|
}
|
|
function groupByProject() {
|
|
groupWarnings(generateSectionsByProject);
|
|
}
|
|
"""
|
|
|
|
|
|
# Emit a JavaScript const string
|
|
def emit_const_string(name, value):
|
|
print('const ' + name + ' = "' + escape_string(value) + '";')
|
|
|
|
|
|
# Emit a JavaScript const integer array.
|
|
def emit_const_int_array(name, array):
|
|
print('const ' + name + ' = [')
|
|
for n in array:
|
|
print(str(n) + ',')
|
|
print('];')
|
|
|
|
|
|
# Emit a JavaScript const string array.
|
|
def emit_const_string_array(name, array):
|
|
print('const ' + name + ' = [')
|
|
for s in array:
|
|
print('"' + strip_escape_string(s) + '",')
|
|
print('];')
|
|
|
|
|
|
# Emit a JavaScript const string array for HTML.
|
|
def emit_const_html_string_array(name, array):
|
|
print('const ' + name + ' = [')
|
|
for s in array:
|
|
# Not using html.escape yet, to work for both python 2 and 3,
|
|
# until all users switch to python 3.
|
|
# pylint:disable=deprecated-method
|
|
print('"' + cgi.escape(strip_escape_string(s)) + '",')
|
|
print('];')
|
|
|
|
|
|
# Emit a JavaScript const object array.
|
|
def emit_const_object_array(name, array):
|
|
print('const ' + name + ' = [')
|
|
for x in array:
|
|
print(str(x) + ',')
|
|
print('];')
|
|
|
|
|
|
def emit_js_data():
|
|
"""Dump dynamic HTML page's static JavaScript data."""
|
|
emit_const_string('FlagURL', args.url if args.url else '')
|
|
emit_const_string('FlagSeparator', args.separator if args.separator else '')
|
|
emit_const_string_array('SeverityColors', [s.color for s in Severity.levels])
|
|
emit_const_string_array('SeverityHeaders',
|
|
[s.header for s in Severity.levels])
|
|
emit_const_string_array('SeverityColumnHeaders',
|
|
[s.column_header for s in Severity.levels])
|
|
emit_const_string_array('ProjectNames', project_names)
|
|
# pytype: disable=attribute-error
|
|
emit_const_int_array('WarnPatternsSeverity',
|
|
[w['severity'].value for w in warn_patterns])
|
|
# pytype: enable=attribute-error
|
|
emit_const_html_string_array('WarnPatternsDescription',
|
|
[w['description'] for w in warn_patterns])
|
|
emit_const_html_string_array('WarnPatternsOption',
|
|
[w['option'] for w in warn_patterns])
|
|
emit_const_html_string_array('WarningMessages', warning_messages)
|
|
emit_const_object_array('Warnings', warning_records)
|
|
|
|
draw_table_javascript = """
|
|
google.charts.load('current', {'packages':['table']});
|
|
google.charts.setOnLoadCallback(drawTable);
|
|
function drawTable() {
|
|
var data = new google.visualization.DataTable();
|
|
data.addColumn('string', StatsHeader[0]);
|
|
for (var i=1; i<StatsHeader.length; i++) {
|
|
data.addColumn('number', StatsHeader[i]);
|
|
}
|
|
data.addRows(StatsRows);
|
|
for (var i=0; i<StatsRows.length; i++) {
|
|
for (var j=0; j<StatsHeader.length; j++) {
|
|
data.setProperty(i, j, 'style', 'border:1px solid black;');
|
|
}
|
|
}
|
|
var table = new google.visualization.Table(document.getElementById('stats_table'));
|
|
table.draw(data, {allowHtml: true, alternatingRowStyle: true});
|
|
}
|
|
"""
|
|
|
|
|
|
def dump_html():
|
|
"""Dump the html output to stdout."""
|
|
dump_html_prologue('Warnings for ' + platform_version + ' - ' +
|
|
target_product + ' - ' + target_variant)
|
|
dump_stats()
|
|
print('<br><div id="stats_table"></div><br>')
|
|
print('\n<script>')
|
|
emit_js_data()
|
|
print(scripts_for_warning_groups)
|
|
print('</script>')
|
|
emit_buttons()
|
|
# Warning messages are grouped by severities or project names.
|
|
print('<br><div id="warning_groups"></div>')
|
|
if args.byproject:
|
|
print('<script>groupByProject();</script>')
|
|
else:
|
|
print('<script>groupBySeverity();</script>')
|
|
dump_fixed()
|
|
dump_html_epilogue()
|
|
|
|
|
|
##### Functions to count warnings and dump csv file. #########################
|
|
|
|
|
|
def description_for_csv(category):
|
|
if not category['description']:
|
|
return '?'
|
|
return category['description']
|
|
|
|
|
|
def count_severity(writer, sev, kind):
|
|
"""Count warnings of given severity."""
|
|
total = 0
|
|
for i in warn_patterns:
|
|
if i['severity'] == sev and i['members']:
|
|
n = len(i['members'])
|
|
total += n
|
|
warning = kind + ': ' + description_for_csv(i)
|
|
writer.writerow([n, '', warning])
|
|
# print number of warnings for each project, ordered by project name.
|
|
# pytype: disable=attribute-error
|
|
projects = sorted(i['projects'].keys())
|
|
# pytype: enable=attribute-error
|
|
for p in projects:
|
|
writer.writerow([i['projects'][p], p, warning])
|
|
writer.writerow([total, '', kind + ' warnings'])
|
|
|
|
return total
|
|
|
|
|
|
# dump number of warnings in csv format to stdout
|
|
def dump_csv(writer):
|
|
"""Dump number of warnings in csv format to stdout."""
|
|
sort_warnings()
|
|
total = 0
|
|
for s in Severity.levels:
|
|
if s != Severity.SEVERITY_UNKNOWN:
|
|
total += count_severity(writer, s, s.column_header)
|
|
writer.writerow([total, '', 'All warnings'])
|
|
|
|
|
|
def common_main(parallel_process):
|
|
"""Real main function to classify warnings and generate .html file."""
|
|
find_android_root()
|
|
# We must use 'utf-8' codec to parse some non-ASCII code in warnings.
|
|
warning_lines = parse_input_file(
|
|
io.open(args.buildlog, mode='r', encoding='utf-8'))
|
|
parallel_classify_warnings(warning_lines, parallel_process)
|
|
# If a user pases a csv path, save the fileoutput to the path
|
|
# If the user also passed gencsv write the output to stdout
|
|
# If the user did not pass gencsv flag dump the html report to stdout.
|
|
if args.csvpath:
|
|
with open(args.csvpath, 'w') as f:
|
|
dump_csv(csv.writer(f, lineterminator='\n'))
|
|
if args.gencsv:
|
|
dump_csv(csv.writer(sys.stdout, lineterminator='\n'))
|
|
else:
|
|
dump_html()
|