filter-known-issues: add warning support, clean up, add more help
Introduce support to tag some issues as warnings, that can be printed but not considered errors. Added the ability to write the errors and warnings to split files, for later reporting. Added a more consistent help. Change-Id: Ia75430c3337385afca59595437168ab643c92911 Signed-off-by: Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>
This commit is contained in:
Inaky Perez-Gonzalez
Anas Nashif
parent
110d58c055
commit
e45f05bc1e
1 changed files with 128 additions and 41 deletions
|
|
@ -1,37 +1,55 @@
|
|||
#! /usr/bin/env python2
|
||||
"""
|
||||
Filters a file, classifying output in errors, warnings and discarding
|
||||
the rest.
|
||||
|
||||
Given a set of regular expresions read from files named *.conf in the
|
||||
given configuration path(s), of the format:
|
||||
|
||||
#
|
||||
# Comments for multiline regex 1...
|
||||
#
|
||||
MULTILINEPYTHONREGEX
|
||||
MULTILINEPYTHONREGEX
|
||||
MULTILINEPYTHONREGEX
|
||||
#
|
||||
# Comments for multiline regex 2...
|
||||
#
|
||||
#WARNING
|
||||
MULTILINEPYTHONREGEX2
|
||||
MULTILINEPYTHONREGEX2
|
||||
|
||||
Anything matched by MULTILINEPYTHONREGEX will be considered something
|
||||
to be filtered out and not printed.
|
||||
|
||||
Anything matched by MULTILINEPYHONREGEX with a #WARNING tag in the
|
||||
comment means (optional) means that it describes something that is
|
||||
considered to be a warning. Print it to stderr.
|
||||
|
||||
Anything leftover is considred to be errors, printed to stdout.
|
||||
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import mmap
|
||||
import os
|
||||
import re
|
||||
import sre_constants
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
exclude_regexs = []
|
||||
|
||||
noncomment_regex = re.compile(
|
||||
"(^[ \t][^#]+.*\n)+"
|
||||
, re.MULTILINE)
|
||||
# first is a list of one or more comment lines
|
||||
# followed by a list of non-comments which describe a multiline regex
|
||||
config_regex = \
|
||||
"(?P<comment>(^\s*#.*\n)+)" \
|
||||
"(?P<regex>(^[^#].*\n)+)"
|
||||
|
||||
def config_import_file(filename):
|
||||
"""
|
||||
Imports regular expresions from any file *.conf in the given path
|
||||
|
||||
Each file follows the format::
|
||||
|
||||
#
|
||||
# Comments for multiline regex 1...
|
||||
#
|
||||
multilineregex
|
||||
multilineregex
|
||||
multilineregex
|
||||
#
|
||||
# Comments for multiline regex 2...
|
||||
#
|
||||
multilineregex
|
||||
multilineregex
|
||||
multilineregex
|
||||
|
||||
etc.
|
||||
Imports regular expresions from any file *.conf in the given path,
|
||||
format as given in the main doc
|
||||
"""
|
||||
try:
|
||||
with open(filename, "rb") as f:
|
||||
|
|
@ -41,21 +59,27 @@ def config_import_file(filename):
|
|||
# finditer() finds all the blocks and selects
|
||||
# the bits of mmapped-file that comprises
|
||||
# each--we compile it into a regex and append.
|
||||
for m in re.finditer("(^\s*[^#].*\n)+", mm, re.MULTILINE):
|
||||
for m in re.finditer(config_regex, mm, re.MULTILINE):
|
||||
origin = "%s:%s-%s" % (filename, m.start(), m.end())
|
||||
gd = m.groupdict()
|
||||
comment = gd['comment']
|
||||
regex = gd['regex']
|
||||
try:
|
||||
r = re.compile(mm[m.start():m.end()], re.MULTILINE)
|
||||
except Exception as e:
|
||||
r = re.compile(regex, re.MULTILINE)
|
||||
except sre_constants.error as e:
|
||||
logging.error("%s: bytes %d-%d: bad regex: %s",
|
||||
filename, m.start(), m.end(), e)
|
||||
raise
|
||||
logging.debug("%s: found regex at bytes %d-%d: %s",
|
||||
filename, m.start(), m.end(),
|
||||
mm[m.start():m.end()])
|
||||
exclude_regexs.append((r, origin))
|
||||
filename, m.start(), m.end(), regex)
|
||||
if '#WARNING' in comment:
|
||||
exclude_regexs.append((r, origin, ('warning',)))
|
||||
else:
|
||||
exclude_regexs.append((r, origin, ()))
|
||||
logging.debug("%s: loaded", filename)
|
||||
except Exception as e:
|
||||
raise Exception("E: %s: can't load config file: %s" % (filename, e))
|
||||
logging.error("E: %s: can't load config file: %s" % (filename, e))
|
||||
raise
|
||||
|
||||
def config_import_path(path):
|
||||
"""
|
||||
|
|
@ -71,7 +95,8 @@ def config_import_path(path):
|
|||
continue
|
||||
config_import_file(filename)
|
||||
except Exception as e:
|
||||
raise Exception("E: %s: can't load config files: %s" % (path, e))
|
||||
raise Exception("E: %s: can't load config files: %s %s" % (path,
|
||||
e, traceback.format_exc()))
|
||||
|
||||
def config_import(paths):
|
||||
"""
|
||||
|
|
@ -92,11 +117,17 @@ def config_import(paths):
|
|||
for path in _paths:
|
||||
config_import_path(path)
|
||||
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
arg_parser = argparse.ArgumentParser(
|
||||
description = __doc__,
|
||||
formatter_class = argparse.RawDescriptionHelpFormatter)
|
||||
arg_parser.add_argument("-v", "--verbosity", action = "count", default = 0,
|
||||
help = "increase verbosity")
|
||||
arg_parser.add_argument("-q", "--quiet", action = "count", default = 0,
|
||||
help = "decrease verbosity")
|
||||
arg_parser.add_argument("-e", "--errors", action = "store", default = None,
|
||||
help = "file where to store errors")
|
||||
arg_parser.add_argument("-w", "--warnings", action = "store", default = None,
|
||||
help = "file where to store warnings")
|
||||
arg_parser.add_argument("-c", "--config-dir", action = "append", nargs = "?",
|
||||
default = [ ".known-issues/" ],
|
||||
help = "configuration directory (multiple can be "
|
||||
|
|
@ -115,6 +146,44 @@ config_import(args.config_dir)
|
|||
|
||||
exclude_ranges = []
|
||||
|
||||
if args.warnings:
|
||||
warnings = open(args.warnings, "w")
|
||||
else:
|
||||
warnings = None
|
||||
if args.errors:
|
||||
errors = open(args.errors, "w")
|
||||
else:
|
||||
errors = None
|
||||
|
||||
def report_error(data):
|
||||
sys.stdout.write(data)
|
||||
if errors:
|
||||
errors.write(data)
|
||||
|
||||
def report_warning(data):
|
||||
sys.stderr.write(data)
|
||||
if warnings:
|
||||
warnings.write(data)
|
||||
|
||||
if args.warnings:
|
||||
warnings = open(args.warnings, "w")
|
||||
else:
|
||||
warnings = None
|
||||
if args.errors:
|
||||
errors = open(args.errors, "w")
|
||||
else:
|
||||
errors = None
|
||||
|
||||
def report_error(data):
|
||||
sys.stdout.write(data)
|
||||
if errors:
|
||||
errors.write(data)
|
||||
|
||||
def report_warning(data):
|
||||
sys.stderr.write(data)
|
||||
if warnings:
|
||||
warnings.write(data)
|
||||
|
||||
for filename in args.FILENAMEs:
|
||||
try:
|
||||
with open(filename, "r+b") as f:
|
||||
|
|
@ -122,30 +191,48 @@ for filename in args.FILENAMEs:
|
|||
# Yeah, this should be more protected in case of exception
|
||||
# and such, but this is a short running program...
|
||||
mm = mmap.mmap(f.fileno(), 0)
|
||||
for ex, origin in exclude_regexs:
|
||||
for ex, origin, flags in exclude_regexs:
|
||||
logging.info("%s: searching from %s: %s",
|
||||
filename, origin, ex.pattern)
|
||||
for m in re.finditer(ex.pattern, mm, re.MULTILINE):
|
||||
logging.debug("%s: %s-%s: match from from %s",
|
||||
filename, m.start(), m.end(), origin)
|
||||
exclude_ranges.append((m.start(), m.end()))
|
||||
logging.info("%s: %s-%s: match from from %s %s",
|
||||
filename, m.start(), m.end(), origin, flags)
|
||||
if 'warning' in flags:
|
||||
exclude_ranges.append((m.start(), m.end(), True))
|
||||
else:
|
||||
exclude_ranges.append((m.start(), m.end(), False))
|
||||
|
||||
exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0])
|
||||
logging.warning("%s: ranges excluded: %s", filename, exclude_ranges)
|
||||
|
||||
# Printd what has not been filtered
|
||||
# Decide what to do with what has been filtered; warnings
|
||||
# go to stderr and warnings file, errors to stdout, what
|
||||
# is ignored is just dumped.
|
||||
offset = 0
|
||||
for b, e in exclude_ranges:
|
||||
for b, e, warning in exclude_ranges:
|
||||
mm.seek(offset)
|
||||
d = b - offset
|
||||
logging.debug("%s: exclude range (%d, %d), from %d %dB",
|
||||
filename, b, e, offset, d)
|
||||
if b > offset:
|
||||
print(mm.read(d - 1))
|
||||
# We have something not caught by a filter, an error
|
||||
logging.info("%s: error range (%d, %d), from %d %dB",
|
||||
filename, offset, b, offset, b - offset)
|
||||
report_error(mm.read(b - offset))
|
||||
mm.seek(b)
|
||||
if warning == True: # A warning, print it
|
||||
mm.seek(b)
|
||||
logging.info("%s: warning range (%d, %d), from %d %dB",
|
||||
filename, b, e, offset, e - b)
|
||||
report_warning(mm.read(e - b))
|
||||
else: # Exclude, ignore it
|
||||
d = b - offset
|
||||
logging.info("%s: exclude range (%d, %d), from %d %dB",
|
||||
filename, b, e, offset, d)
|
||||
offset = e
|
||||
mm.seek(offset)
|
||||
if len(mm) != offset:
|
||||
print mm.read(len(mm) - offset - 1)
|
||||
logging.info("%s: error final range (%d, %d), from %d %dB",
|
||||
filename, b, e, offset, e - b)
|
||||
report_error(mm.read(len(mm) - offset - 1))
|
||||
del mm
|
||||
except Exception as e:
|
||||
logging.error("%s: cannot load: %s", filename, e)
|
||||
raise
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue