forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
471 lines
15 KiB
471 lines
15 KiB
#!/usr/bin/env python3 |
|
# SPDX-License-Identifier: GPL-2.0-only |
|
|
|
"""Find Kconfig symbols that are referenced but not defined.""" |
|
|
|
# (c) 2014-2017 Valentin Rothberg <[email protected]> |
|
# (c) 2014 Stefan Hengelein <[email protected]> |
|
# |
|
|
|
|
|
import argparse |
|
import difflib |
|
import os |
|
import re |
|
import signal |
|
import subprocess |
|
import sys |
|
from multiprocessing import Pool, cpu_count |
|
|
|
|
|
# regex expressions |
|
OPERATORS = r"&|\(|\)|\||\!" |
|
SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}" |
|
DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*" |
|
EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+" |
|
DEFAULT = r"default\s+.*?(?:if\s.+){,1}" |
|
STMT = r"^\s*(?:if|select|imply|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR |
|
SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")" |
|
|
|
# regex objects |
|
REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
|
REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)') |
|
REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL) |
|
REGEX_KCONFIG_DEF = re.compile(DEF) |
|
REGEX_KCONFIG_EXPR = re.compile(EXPR) |
|
REGEX_KCONFIG_STMT = re.compile(STMT) |
|
REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$") |
|
REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
|
REGEX_QUOTES = re.compile("(\"(.*?)\")") |
|
|
|
|
|
def parse_options(): |
|
"""The user interface of this module.""" |
|
usage = "Run this tool to detect Kconfig symbols that are referenced but " \ |
|
"not defined in Kconfig. If no option is specified, " \ |
|
"checkkconfigsymbols defaults to check your current tree. " \ |
|
"Please note that specifying commits will 'git reset --hard\' " \ |
|
"your current tree! You may save uncommitted changes to avoid " \ |
|
"losing data." |
|
|
|
parser = argparse.ArgumentParser(description=usage) |
|
|
|
parser.add_argument('-c', '--commit', dest='commit', action='store', |
|
default="", |
|
help="check if the specified commit (hash) introduces " |
|
"undefined Kconfig symbols") |
|
|
|
parser.add_argument('-d', '--diff', dest='diff', action='store', |
|
default="", |
|
help="diff undefined symbols between two commits " |
|
"(e.g., -d commmit1..commit2)") |
|
|
|
parser.add_argument('-f', '--find', dest='find', action='store_true', |
|
default=False, |
|
help="find and show commits that may cause symbols to be " |
|
"missing (required to run with --diff)") |
|
|
|
parser.add_argument('-i', '--ignore', dest='ignore', action='store', |
|
default="", |
|
help="ignore files matching this Python regex " |
|
"(e.g., -i '.*defconfig')") |
|
|
|
parser.add_argument('-s', '--sim', dest='sim', action='store', default="", |
|
help="print a list of max. 10 string-similar symbols") |
|
|
|
parser.add_argument('--force', dest='force', action='store_true', |
|
default=False, |
|
help="reset current Git tree even when it's dirty") |
|
|
|
parser.add_argument('--no-color', dest='color', action='store_false', |
|
default=True, |
|
help="don't print colored output (default when not " |
|
"outputting to a terminal)") |
|
|
|
args = parser.parse_args() |
|
|
|
if args.commit and args.diff: |
|
sys.exit("Please specify only one option at once.") |
|
|
|
if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff): |
|
sys.exit("Please specify valid input in the following format: " |
|
"\'commit1..commit2\'") |
|
|
|
if args.commit or args.diff: |
|
if not args.force and tree_is_dirty(): |
|
sys.exit("The current Git tree is dirty (see 'git status'). " |
|
"Running this script may\ndelete important data since it " |
|
"calls 'git reset --hard' for some performance\nreasons. " |
|
" Please run this script in a clean Git tree or pass " |
|
"'--force' if you\nwant to ignore this warning and " |
|
"continue.") |
|
|
|
if args.commit: |
|
if args.commit.startswith('HEAD'): |
|
sys.exit("The --commit option can't use the HEAD ref") |
|
|
|
args.find = False |
|
|
|
if args.ignore: |
|
try: |
|
re.match(args.ignore, "this/is/just/a/test.c") |
|
except: |
|
sys.exit("Please specify a valid Python regex.") |
|
|
|
return args |
|
|
|
|
|
def main(): |
|
"""Main function of this module.""" |
|
args = parse_options() |
|
|
|
global COLOR |
|
COLOR = args.color and sys.stdout.isatty() |
|
|
|
if args.sim and not args.commit and not args.diff: |
|
sims = find_sims(args.sim, args.ignore) |
|
if sims: |
|
print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) |
|
else: |
|
print("%s: no similar symbols found" % yel("Similar symbols")) |
|
sys.exit(0) |
|
|
|
# dictionary of (un)defined symbols |
|
defined = {} |
|
undefined = {} |
|
|
|
if args.commit or args.diff: |
|
head = get_head() |
|
|
|
# get commit range |
|
commit_a = None |
|
commit_b = None |
|
if args.commit: |
|
commit_a = args.commit + "~" |
|
commit_b = args.commit |
|
elif args.diff: |
|
split = args.diff.split("..") |
|
commit_a = split[0] |
|
commit_b = split[1] |
|
undefined_a = {} |
|
undefined_b = {} |
|
|
|
# get undefined items before the commit |
|
reset(commit_a) |
|
undefined_a, _ = check_symbols(args.ignore) |
|
|
|
# get undefined items for the commit |
|
reset(commit_b) |
|
undefined_b, defined = check_symbols(args.ignore) |
|
|
|
# report cases that are present for the commit but not before |
|
for symbol in sorted(undefined_b): |
|
# symbol has not been undefined before |
|
if symbol not in undefined_a: |
|
files = sorted(undefined_b.get(symbol)) |
|
undefined[symbol] = files |
|
# check if there are new files that reference the undefined symbol |
|
else: |
|
files = sorted(undefined_b.get(symbol) - |
|
undefined_a.get(symbol)) |
|
if files: |
|
undefined[symbol] = files |
|
|
|
# reset to head |
|
reset(head) |
|
|
|
# default to check the entire tree |
|
else: |
|
undefined, defined = check_symbols(args.ignore) |
|
|
|
# now print the output |
|
for symbol in sorted(undefined): |
|
print(red(symbol)) |
|
|
|
files = sorted(undefined.get(symbol)) |
|
print("%s: %s" % (yel("Referencing files"), ", ".join(files))) |
|
|
|
sims = find_sims(symbol, args.ignore, defined) |
|
sims_out = yel("Similar symbols") |
|
if sims: |
|
print("%s: %s" % (sims_out, ', '.join(sims))) |
|
else: |
|
print("%s: %s" % (sims_out, "no similar symbols found")) |
|
|
|
if args.find: |
|
print("%s:" % yel("Commits changing symbol")) |
|
commits = find_commits(symbol, args.diff) |
|
if commits: |
|
for commit in commits: |
|
commit = commit.split(" ", 1) |
|
print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) |
|
else: |
|
print("\t- no commit found") |
|
print() # new line |
|
|
|
|
|
def reset(commit): |
|
"""Reset current git tree to %commit.""" |
|
execute(["git", "reset", "--hard", commit]) |
|
|
|
|
|
def yel(string): |
|
""" |
|
Color %string yellow. |
|
""" |
|
return "\033[33m%s\033[0m" % string if COLOR else string |
|
|
|
|
|
def red(string): |
|
""" |
|
Color %string red. |
|
""" |
|
return "\033[31m%s\033[0m" % string if COLOR else string |
|
|
|
|
|
def execute(cmd): |
|
"""Execute %cmd and return stdout. Exit in case of error.""" |
|
try: |
|
stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False) |
|
stdout = stdout.decode(errors='replace') |
|
except subprocess.CalledProcessError as fail: |
|
exit(fail) |
|
return stdout |
|
|
|
|
|
def find_commits(symbol, diff): |
|
"""Find commits changing %symbol in the given range of %diff.""" |
|
commits = execute(["git", "log", "--pretty=oneline", |
|
"--abbrev-commit", "-G", |
|
symbol, diff]) |
|
return [x for x in commits.split("\n") if x] |
|
|
|
|
|
def tree_is_dirty(): |
|
"""Return true if the current working tree is dirty (i.e., if any file has |
|
been added, deleted, modified, renamed or copied but not committed).""" |
|
stdout = execute(["git", "status", "--porcelain"]) |
|
for line in stdout: |
|
if re.findall(r"[URMADC]{1}", line[:2]): |
|
return True |
|
return False |
|
|
|
|
|
def get_head(): |
|
"""Return commit hash of current HEAD.""" |
|
stdout = execute(["git", "rev-parse", "HEAD"]) |
|
return stdout.strip('\n') |
|
|
|
|
|
def partition(lst, size): |
|
"""Partition list @lst into eveni-sized lists of size @size.""" |
|
return [lst[i::size] for i in range(size)] |
|
|
|
|
|
def init_worker(): |
|
"""Set signal handler to ignore SIGINT.""" |
|
signal.signal(signal.SIGINT, signal.SIG_IGN) |
|
|
|
|
|
def find_sims(symbol, ignore, defined=[]): |
|
"""Return a list of max. ten Kconfig symbols that are string-similar to |
|
@symbol.""" |
|
if defined: |
|
return difflib.get_close_matches(symbol, set(defined), 10) |
|
|
|
pool = Pool(cpu_count(), init_worker) |
|
kfiles = [] |
|
for gitfile in get_files(): |
|
if REGEX_FILE_KCONFIG.match(gitfile): |
|
kfiles.append(gitfile) |
|
|
|
arglist = [] |
|
for part in partition(kfiles, cpu_count()): |
|
arglist.append((part, ignore)) |
|
|
|
for res in pool.map(parse_kconfig_files, arglist): |
|
defined.extend(res[0]) |
|
|
|
return difflib.get_close_matches(symbol, set(defined), 10) |
|
|
|
|
|
def get_files(): |
|
"""Return a list of all files in the current git directory.""" |
|
# use 'git ls-files' to get the worklist |
|
stdout = execute(["git", "ls-files"]) |
|
if len(stdout) > 0 and stdout[-1] == "\n": |
|
stdout = stdout[:-1] |
|
|
|
files = [] |
|
for gitfile in stdout.rsplit("\n"): |
|
if ".git" in gitfile or "ChangeLog" in gitfile or \ |
|
".log" in gitfile or os.path.isdir(gitfile) or \ |
|
gitfile.startswith("tools/"): |
|
continue |
|
files.append(gitfile) |
|
return files |
|
|
|
|
|
def check_symbols(ignore): |
|
"""Find undefined Kconfig symbols and return a dict with the symbol as key |
|
and a list of referencing files as value. Files matching %ignore are not |
|
checked for undefined symbols.""" |
|
pool = Pool(cpu_count(), init_worker) |
|
try: |
|
return check_symbols_helper(pool, ignore) |
|
except KeyboardInterrupt: |
|
pool.terminate() |
|
pool.join() |
|
sys.exit(1) |
|
|
|
|
|
def check_symbols_helper(pool, ignore): |
|
"""Helper method for check_symbols(). Used to catch keyboard interrupts in |
|
check_symbols() in order to properly terminate running worker processes.""" |
|
source_files = [] |
|
kconfig_files = [] |
|
defined_symbols = [] |
|
referenced_symbols = dict() # {file: [symbols]} |
|
|
|
for gitfile in get_files(): |
|
if REGEX_FILE_KCONFIG.match(gitfile): |
|
kconfig_files.append(gitfile) |
|
else: |
|
if ignore and re.match(ignore, gitfile): |
|
continue |
|
# add source files that do not match the ignore pattern |
|
source_files.append(gitfile) |
|
|
|
# parse source files |
|
arglist = partition(source_files, cpu_count()) |
|
for res in pool.map(parse_source_files, arglist): |
|
referenced_symbols.update(res) |
|
|
|
# parse kconfig files |
|
arglist = [] |
|
for part in partition(kconfig_files, cpu_count()): |
|
arglist.append((part, ignore)) |
|
for res in pool.map(parse_kconfig_files, arglist): |
|
defined_symbols.extend(res[0]) |
|
referenced_symbols.update(res[1]) |
|
defined_symbols = set(defined_symbols) |
|
|
|
# inverse mapping of referenced_symbols to dict(symbol: [files]) |
|
inv_map = dict() |
|
for _file, symbols in referenced_symbols.items(): |
|
for symbol in symbols: |
|
inv_map[symbol] = inv_map.get(symbol, set()) |
|
inv_map[symbol].add(_file) |
|
referenced_symbols = inv_map |
|
|
|
undefined = {} # {symbol: [files]} |
|
for symbol in sorted(referenced_symbols): |
|
# filter some false positives |
|
if symbol == "FOO" or symbol == "BAR" or \ |
|
symbol == "FOO_BAR" or symbol == "XXX": |
|
continue |
|
if symbol not in defined_symbols: |
|
if symbol.endswith("_MODULE"): |
|
# avoid false positives for kernel modules |
|
if symbol[:-len("_MODULE")] in defined_symbols: |
|
continue |
|
undefined[symbol] = referenced_symbols.get(symbol) |
|
return undefined, defined_symbols |
|
|
|
|
|
def parse_source_files(source_files): |
|
"""Parse each source file in @source_files and return dictionary with source |
|
files as keys and lists of references Kconfig symbols as values.""" |
|
referenced_symbols = dict() |
|
for sfile in source_files: |
|
referenced_symbols[sfile] = parse_source_file(sfile) |
|
return referenced_symbols |
|
|
|
|
|
def parse_source_file(sfile): |
|
"""Parse @sfile and return a list of referenced Kconfig symbols.""" |
|
lines = [] |
|
references = [] |
|
|
|
if not os.path.exists(sfile): |
|
return references |
|
|
|
with open(sfile, "r", encoding='utf-8', errors='replace') as stream: |
|
lines = stream.readlines() |
|
|
|
for line in lines: |
|
if "CONFIG_" not in line: |
|
continue |
|
symbols = REGEX_SOURCE_SYMBOL.findall(line) |
|
for symbol in symbols: |
|
if not REGEX_FILTER_SYMBOLS.search(symbol): |
|
continue |
|
references.append(symbol) |
|
|
|
return references |
|
|
|
|
|
def get_symbols_in_line(line): |
|
"""Return mentioned Kconfig symbols in @line.""" |
|
return REGEX_SYMBOL.findall(line) |
|
|
|
|
|
def parse_kconfig_files(args): |
|
"""Parse kconfig files and return tuple of defined and references Kconfig |
|
symbols. Note, @args is a tuple of a list of files and the @ignore |
|
pattern.""" |
|
kconfig_files = args[0] |
|
ignore = args[1] |
|
defined_symbols = [] |
|
referenced_symbols = dict() |
|
|
|
for kfile in kconfig_files: |
|
defined, references = parse_kconfig_file(kfile) |
|
defined_symbols.extend(defined) |
|
if ignore and re.match(ignore, kfile): |
|
# do not collect references for files that match the ignore pattern |
|
continue |
|
referenced_symbols[kfile] = references |
|
return (defined_symbols, referenced_symbols) |
|
|
|
|
|
def parse_kconfig_file(kfile): |
|
"""Parse @kfile and update symbol definitions and references.""" |
|
lines = [] |
|
defined = [] |
|
references = [] |
|
|
|
if not os.path.exists(kfile): |
|
return defined, references |
|
|
|
with open(kfile, "r", encoding='utf-8', errors='replace') as stream: |
|
lines = stream.readlines() |
|
|
|
for i in range(len(lines)): |
|
line = lines[i] |
|
line = line.strip('\n') |
|
line = line.split("#")[0] # ignore comments |
|
|
|
if REGEX_KCONFIG_DEF.match(line): |
|
symbol_def = REGEX_KCONFIG_DEF.findall(line) |
|
defined.append(symbol_def[0]) |
|
elif REGEX_KCONFIG_STMT.match(line): |
|
line = REGEX_QUOTES.sub("", line) |
|
symbols = get_symbols_in_line(line) |
|
# multi-line statements |
|
while line.endswith("\\"): |
|
i += 1 |
|
line = lines[i] |
|
line = line.strip('\n') |
|
symbols.extend(get_symbols_in_line(line)) |
|
for symbol in set(symbols): |
|
if REGEX_NUMERIC.match(symbol): |
|
# ignore numeric values |
|
continue |
|
references.append(symbol) |
|
|
|
return defined, references |
|
|
|
|
|
if __name__ == "__main__": |
|
main()
|
|
|