mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
447 lines
14 KiB
447 lines
14 KiB
#!/usr/bin/env python2 |
|
# SPDX-License-Identifier: GPL-2.0+ |
|
# |
|
# Author: Masahiro Yamada <[email protected]> |
|
# |
|
|
|
""" |
|
Converter from Kconfig and MAINTAINERS to a board database. |
|
|
|
Run 'tools/genboardscfg.py' to create a board database. |
|
|
|
Run 'tools/genboardscfg.py -h' for available options. |
|
|
|
Python 2.6 or later, but not Python 3.x is necessary to run this script. |
|
""" |
|
|
|
import errno |
|
import fnmatch |
|
import glob |
|
import multiprocessing |
|
import optparse |
|
import os |
|
import sys |
|
import tempfile |
|
import time |
|
|
|
sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman')) |
|
import kconfiglib |
|
|
|
### constant variables ### |
|
OUTPUT_FILE = 'boards.cfg' |
|
CONFIG_DIR = 'configs' |
|
SLEEP_TIME = 0.03 |
|
COMMENT_BLOCK = '''# |
|
# List of boards |
|
# Automatically generated by %s: don't edit |
|
# |
|
# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers |
|
|
|
''' % __file__ |
|
|
|
### helper functions ### |
|
def try_remove(f): |
|
"""Remove a file ignoring 'No such file or directory' error.""" |
|
try: |
|
os.remove(f) |
|
except OSError as exception: |
|
# Ignore 'No such file or directory' error |
|
if exception.errno != errno.ENOENT: |
|
raise |
|
|
|
def check_top_directory(): |
|
"""Exit if we are not at the top of source directory.""" |
|
for f in ('README', 'Licenses'): |
|
if not os.path.exists(f): |
|
sys.exit('Please run at the top of source directory.') |
|
|
|
def output_is_new(output): |
|
"""Check if the output file is up to date. |
|
|
|
Returns: |
|
True if the given output file exists and is newer than any of |
|
*_defconfig, MAINTAINERS and Kconfig*. False otherwise. |
|
""" |
|
try: |
|
ctime = os.path.getctime(output) |
|
except OSError as exception: |
|
if exception.errno == errno.ENOENT: |
|
# return False on 'No such file or directory' error |
|
return False |
|
else: |
|
raise |
|
|
|
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): |
|
for filename in fnmatch.filter(filenames, '*_defconfig'): |
|
if fnmatch.fnmatch(filename, '.*'): |
|
continue |
|
filepath = os.path.join(dirpath, filename) |
|
if ctime < os.path.getctime(filepath): |
|
return False |
|
|
|
for (dirpath, dirnames, filenames) in os.walk('.'): |
|
for filename in filenames: |
|
if (fnmatch.fnmatch(filename, '*~') or |
|
not fnmatch.fnmatch(filename, 'Kconfig*') and |
|
not filename == 'MAINTAINERS'): |
|
continue |
|
filepath = os.path.join(dirpath, filename) |
|
if ctime < os.path.getctime(filepath): |
|
return False |
|
|
|
# Detect a board that has been removed since the current board database |
|
# was generated |
|
with open(output) as f: |
|
for line in f: |
|
if line[0] == '#' or line == '\n': |
|
continue |
|
defconfig = line.split()[6] + '_defconfig' |
|
if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): |
|
return False |
|
|
|
return True |
|
|
|
### classes ### |
|
class KconfigScanner: |
|
|
|
"""Kconfig scanner.""" |
|
|
|
### constant variable only used in this class ### |
|
_SYMBOL_TABLE = { |
|
'arch' : 'SYS_ARCH', |
|
'cpu' : 'SYS_CPU', |
|
'soc' : 'SYS_SOC', |
|
'vendor' : 'SYS_VENDOR', |
|
'board' : 'SYS_BOARD', |
|
'config' : 'SYS_CONFIG_NAME', |
|
'options' : 'SYS_EXTRA_OPTIONS' |
|
} |
|
|
|
def __init__(self): |
|
"""Scan all the Kconfig files and create a Config object.""" |
|
# Define environment variables referenced from Kconfig |
|
os.environ['srctree'] = os.getcwd() |
|
os.environ['UBOOTVERSION'] = 'dummy' |
|
os.environ['KCONFIG_OBJDIR'] = '' |
|
self._conf = kconfiglib.Config(print_warnings=False) |
|
|
|
def __del__(self): |
|
"""Delete a leftover temporary file before exit. |
|
|
|
The scan() method of this class creates a temporay file and deletes |
|
it on success. If scan() method throws an exception on the way, |
|
the temporary file might be left over. In that case, it should be |
|
deleted in this destructor. |
|
""" |
|
if hasattr(self, '_tmpfile') and self._tmpfile: |
|
try_remove(self._tmpfile) |
|
|
|
def scan(self, defconfig): |
|
"""Load a defconfig file to obtain board parameters. |
|
|
|
Arguments: |
|
defconfig: path to the defconfig file to be processed |
|
|
|
Returns: |
|
A dictionary of board parameters. It has a form of: |
|
{ |
|
'arch': <arch_name>, |
|
'cpu': <cpu_name>, |
|
'soc': <soc_name>, |
|
'vendor': <vendor_name>, |
|
'board': <board_name>, |
|
'target': <target_name>, |
|
'config': <config_header_name>, |
|
'options': <extra_options> |
|
} |
|
""" |
|
# strip special prefixes and save it in a temporary file |
|
fd, self._tmpfile = tempfile.mkstemp() |
|
with os.fdopen(fd, 'w') as f: |
|
for line in open(defconfig): |
|
colon = line.find(':CONFIG_') |
|
if colon == -1: |
|
f.write(line) |
|
else: |
|
f.write(line[colon + 1:]) |
|
|
|
warnings = self._conf.load_config(self._tmpfile) |
|
if warnings: |
|
for warning in warnings: |
|
print '%s: %s' % (defconfig, warning) |
|
|
|
try_remove(self._tmpfile) |
|
self._tmpfile = None |
|
|
|
params = {} |
|
|
|
# Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc. |
|
# Set '-' if the value is empty. |
|
for key, symbol in self._SYMBOL_TABLE.items(): |
|
value = self._conf.get_symbol(symbol).get_value() |
|
if value: |
|
params[key] = value |
|
else: |
|
params[key] = '-' |
|
|
|
defconfig = os.path.basename(defconfig) |
|
params['target'], match, rear = defconfig.partition('_defconfig') |
|
assert match and not rear, '%s : invalid defconfig' % defconfig |
|
|
|
# fix-up for aarch64 |
|
if params['arch'] == 'arm' and params['cpu'] == 'armv8': |
|
params['arch'] = 'aarch64' |
|
|
|
# fix-up options field. It should have the form: |
|
# <config name>[:comma separated config options] |
|
if params['options'] != '-': |
|
params['options'] = params['config'] + ':' + \ |
|
params['options'].replace(r'\"', '"') |
|
elif params['config'] != params['target']: |
|
params['options'] = params['config'] |
|
|
|
return params |
|
|
|
def scan_defconfigs_for_multiprocess(queue, defconfigs): |
|
"""Scan defconfig files and queue their board parameters |
|
|
|
This function is intended to be passed to |
|
multiprocessing.Process() constructor. |
|
|
|
Arguments: |
|
queue: An instance of multiprocessing.Queue(). |
|
The resulting board parameters are written into it. |
|
defconfigs: A sequence of defconfig files to be scanned. |
|
""" |
|
kconf_scanner = KconfigScanner() |
|
for defconfig in defconfigs: |
|
queue.put(kconf_scanner.scan(defconfig)) |
|
|
|
def read_queues(queues, params_list): |
|
"""Read the queues and append the data to the paramers list""" |
|
for q in queues: |
|
while not q.empty(): |
|
params_list.append(q.get()) |
|
|
|
def scan_defconfigs(jobs=1): |
|
"""Collect board parameters for all defconfig files. |
|
|
|
This function invokes multiple processes for faster processing. |
|
|
|
Arguments: |
|
jobs: The number of jobs to run simultaneously |
|
""" |
|
all_defconfigs = [] |
|
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): |
|
for filename in fnmatch.filter(filenames, '*_defconfig'): |
|
if fnmatch.fnmatch(filename, '.*'): |
|
continue |
|
all_defconfigs.append(os.path.join(dirpath, filename)) |
|
|
|
total_boards = len(all_defconfigs) |
|
processes = [] |
|
queues = [] |
|
for i in range(jobs): |
|
defconfigs = all_defconfigs[total_boards * i / jobs : |
|
total_boards * (i + 1) / jobs] |
|
q = multiprocessing.Queue(maxsize=-1) |
|
p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess, |
|
args=(q, defconfigs)) |
|
p.start() |
|
processes.append(p) |
|
queues.append(q) |
|
|
|
# The resulting data should be accumulated to this list |
|
params_list = [] |
|
|
|
# Data in the queues should be retrieved preriodically. |
|
# Otherwise, the queues would become full and subprocesses would get stuck. |
|
while any([p.is_alive() for p in processes]): |
|
read_queues(queues, params_list) |
|
# sleep for a while until the queues are filled |
|
time.sleep(SLEEP_TIME) |
|
|
|
# Joining subprocesses just in case |
|
# (All subprocesses should already have been finished) |
|
for p in processes: |
|
p.join() |
|
|
|
# retrieve leftover data |
|
read_queues(queues, params_list) |
|
|
|
return params_list |
|
|
|
class MaintainersDatabase: |
|
|
|
"""The database of board status and maintainers.""" |
|
|
|
def __init__(self): |
|
"""Create an empty database.""" |
|
self.database = {} |
|
|
|
def get_status(self, target): |
|
"""Return the status of the given board. |
|
|
|
The board status is generally either 'Active' or 'Orphan'. |
|
Display a warning message and return '-' if status information |
|
is not found. |
|
|
|
Returns: |
|
'Active', 'Orphan' or '-'. |
|
""" |
|
if not target in self.database: |
|
print >> sys.stderr, "WARNING: no status info for '%s'" % target |
|
return '-' |
|
|
|
tmp = self.database[target][0] |
|
if tmp.startswith('Maintained'): |
|
return 'Active' |
|
elif tmp.startswith('Supported'): |
|
return 'Active' |
|
elif tmp.startswith('Orphan'): |
|
return 'Orphan' |
|
else: |
|
print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % |
|
(tmp, target)) |
|
return '-' |
|
|
|
def get_maintainers(self, target): |
|
"""Return the maintainers of the given board. |
|
|
|
Returns: |
|
Maintainers of the board. If the board has two or more maintainers, |
|
they are separated with colons. |
|
""" |
|
if not target in self.database: |
|
print >> sys.stderr, "WARNING: no maintainers for '%s'" % target |
|
return '' |
|
|
|
return ':'.join(self.database[target][1]) |
|
|
|
def parse_file(self, file): |
|
"""Parse a MAINTAINERS file. |
|
|
|
Parse a MAINTAINERS file and accumulates board status and |
|
maintainers information. |
|
|
|
Arguments: |
|
file: MAINTAINERS file to be parsed |
|
""" |
|
targets = [] |
|
maintainers = [] |
|
status = '-' |
|
for line in open(file): |
|
# Check also commented maintainers |
|
if line[:3] == '#M:': |
|
line = line[1:] |
|
tag, rest = line[:2], line[2:].strip() |
|
if tag == 'M:': |
|
maintainers.append(rest) |
|
elif tag == 'F:': |
|
# expand wildcard and filter by 'configs/*_defconfig' |
|
for f in glob.glob(rest): |
|
front, match, rear = f.partition('configs/') |
|
if not front and match: |
|
front, match, rear = rear.rpartition('_defconfig') |
|
if match and not rear: |
|
targets.append(front) |
|
elif tag == 'S:': |
|
status = rest |
|
elif line == '\n': |
|
for target in targets: |
|
self.database[target] = (status, maintainers) |
|
targets = [] |
|
maintainers = [] |
|
status = '-' |
|
if targets: |
|
for target in targets: |
|
self.database[target] = (status, maintainers) |
|
|
|
def insert_maintainers_info(params_list): |
|
"""Add Status and Maintainers information to the board parameters list. |
|
|
|
Arguments: |
|
params_list: A list of the board parameters |
|
""" |
|
database = MaintainersDatabase() |
|
for (dirpath, dirnames, filenames) in os.walk('.'): |
|
if 'MAINTAINERS' in filenames: |
|
database.parse_file(os.path.join(dirpath, 'MAINTAINERS')) |
|
|
|
for i, params in enumerate(params_list): |
|
target = params['target'] |
|
params['status'] = database.get_status(target) |
|
params['maintainers'] = database.get_maintainers(target) |
|
params_list[i] = params |
|
|
|
def format_and_output(params_list, output): |
|
"""Write board parameters into a file. |
|
|
|
Columnate the board parameters, sort lines alphabetically, |
|
and then write them to a file. |
|
|
|
Arguments: |
|
params_list: The list of board parameters |
|
output: The path to the output file |
|
""" |
|
FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target', |
|
'options', 'maintainers') |
|
|
|
# First, decide the width of each column |
|
max_length = dict([ (f, 0) for f in FIELDS]) |
|
for params in params_list: |
|
for f in FIELDS: |
|
max_length[f] = max(max_length[f], len(params[f])) |
|
|
|
output_lines = [] |
|
for params in params_list: |
|
line = '' |
|
for f in FIELDS: |
|
# insert two spaces between fields like column -t would |
|
line += ' ' + params[f].ljust(max_length[f]) |
|
output_lines.append(line.strip()) |
|
|
|
# ignore case when sorting |
|
output_lines.sort(key=str.lower) |
|
|
|
with open(output, 'w') as f: |
|
f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') |
|
|
|
def gen_boards_cfg(output, jobs=1, force=False): |
|
"""Generate a board database file. |
|
|
|
Arguments: |
|
output: The name of the output file |
|
jobs: The number of jobs to run simultaneously |
|
force: Force to generate the output even if it is new |
|
""" |
|
check_top_directory() |
|
|
|
if not force and output_is_new(output): |
|
print "%s is up to date. Nothing to do." % output |
|
sys.exit(0) |
|
|
|
params_list = scan_defconfigs(jobs) |
|
insert_maintainers_info(params_list) |
|
format_and_output(params_list, output) |
|
|
|
def main(): |
|
try: |
|
cpu_count = multiprocessing.cpu_count() |
|
except NotImplementedError: |
|
cpu_count = 1 |
|
|
|
parser = optparse.OptionParser() |
|
# Add options here |
|
parser.add_option('-f', '--force', action="store_true", default=False, |
|
help='regenerate the output even if it is new') |
|
parser.add_option('-j', '--jobs', type='int', default=cpu_count, |
|
help='the number of jobs to run simultaneously') |
|
parser.add_option('-o', '--output', default=OUTPUT_FILE, |
|
help='output file [default=%s]' % OUTPUT_FILE) |
|
(options, args) = parser.parse_args() |
|
|
|
gen_boards_cfg(options.output, jobs=options.jobs, force=options.force) |
|
|
|
if __name__ == '__main__': |
|
main()
|
|
|