From 4041174dc030925a69b05c5caa8dec17a1b9154d Mon Sep 17 00:00:00 2001
From: Adam Števko <xen0l@users.noreply.github.com>
Date: Fri, 19 Oct 2018 19:15:41 +0200
Subject: [PATCH] Generate a component mapping file (#4534)
---
tools/userland-mangler | 4
tools/userland-mapping | 111 +++++++++++++++++++++++++++++++++++++
make-rules/shared-macros.mk | 7 ++
components/Makefile | 11 +++
4 files changed, 129 insertions(+), 4 deletions(-)
diff --git a/components/Makefile b/components/Makefile
index 87f38be..2c7a446 100644
--- a/components/Makefile
+++ b/components/Makefile
@@ -52,6 +52,12 @@
-include $(WS_TOP)/components/$(ENCUMBERED)depends.mk
+# component_mapping.json is auto-generated by the build tools. It provides a map
+# between component path and component FMRI.
+$(WS_TOP)/components/mapping.json:
+ @echo "Generating component mapping..."
+ @$(WS_TOOLS)/userland-mapping --workspace=$(WS_TOP)
+mapping.json: $(WS_TOP)/components/mapping.json
download: TARGET = download
prep: TARGET = prep
@@ -82,7 +88,10 @@
component-hook: $(COMPONENT_DIRS.nosetup)
clean: $(COMPONENT_DIRS.nosetup)
- $(RM) $(WS_TOP)/components/$(ENCUMBERED)components.mk $(WS_TOP)/components/$(ENCUMBERED)depends.mk .profile
+ $(RM) $(WS_TOP)/components/$(ENCUMBERED)components.mk \
+ $(WS_TOP)/components/$(ENCUMBERED)depends.mk \
+ $(WS_TOP)/components/mapping.json \
+ .profile
clobber: $(COMPONENT_DIRS.nosetup) clean
@cd $(WS_TOP)/tools ; echo "clobbering tools..." ; $(GMAKE) clobber
diff --git a/make-rules/shared-macros.mk b/make-rules/shared-macros.mk
index 32d172a..e2b3380 100644
--- a/make-rules/shared-macros.mk
+++ b/make-rules/shared-macros.mk
@@ -647,7 +647,7 @@
FC = $(FC.$(COMPILER).$(BITS))
RUBY_VERSION = 2.3
-RUBY_LIB_VERSION.2.2 = 2.2.0
+RUBY_LIB_VERSION.2.2 = 2.2.0
RUBY_LIB_VERSION.2.3 = 2.3.0
RUBY.2.2 = /usr/ruby/2.2/bin/ruby
RUBY.2.3 = /usr/ruby/2.3/bin/ruby
@@ -1303,3 +1303,8 @@
# is not always what you get.
print-%:
@echo '$(subst ','\'',$*=$($*)) (origin: $(origin $*), flavor: $(flavor $*))'
+
+# A simple rule to print only the value of any macro.
+print-value-%:
+ @echo '$(subst ','\'',$($*))'
+
diff --git a/tools/userland-mangler b/tools/userland-mangler
index 761ac81..c8da3dd 100755
--- a/tools/userland-mangler
+++ b/tools/userland-mangler
@@ -64,7 +64,7 @@
Stability %s"""
attribute_table_footer = """
-.TE
+.TE
.PP
"""
def attributes_section_text(availability, stability, modified_date):
@@ -344,7 +344,7 @@
def main():
import getopt
- # FLUSH STDOUT
+ # FLUSH STDOUT
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
search_paths = []
diff --git a/tools/userland-mapping b/tools/userland-mapping
new file mode 100755
index 0000000..899ccc7
--- /dev/null
+++ b/tools/userland-mapping
@@ -0,0 +1,111 @@
+#!/usr/bin/python2.7
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2018 Adam Stevko
+#
+
+#
+# mapping.py - generate mapping between component FMRI and component package names, to be used for different purposes,
+# e.g. checking for outdated pacakges, vulnerable packages etc.
+#
+
+from __future__ import print_function, absolute_import
+
+import argparse
+import json
+import os
+import re
+import logging
+import subprocess
+import multiprocessing
+import sys
+
+try:
+ from scandir import walk
+except ImportError:
+ from os import walk
+
+logger = logging.getLogger('userland-mapping')
+
+COMONENT_MAPPING_FILENAME = 'mapping.json'
+
+
+def find_component_paths(path, subdir='components', debug=False):
+ expression = re.compile(r'.+\.p5m$', re.IGNORECASE)
+
+ paths = []
+ workspace_path = os.path.join(path, subdir)
+
+ for dirpath, dirnames, filenames in walk(workspace_path):
+ for name in filenames:
+ if expression.match(name):
+ paths.append(dirpath)
+ del dirnames[:]
+ break
+
+ return paths
+
+
+def generate_component_data(component_path):
+ result = []
+
+ proc = subprocess.Popen(['gmake', 'print-value-COMPONENT_NAME', 'print-package-names'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=component_path)
+
+ for out in proc.stdout:
+ result.append(out.rstrip())
+
+ component_name = result[0]
+ component_fmris = result[1:]
+ component_relative_path = component_path.split(os.environ['WS_TOP'])[-1].replace('/', '', 1)
+
+ return component_fmris, component_name, component_relative_path
+
+
+def generate_userland_mapping(workspace_path, subdir='components'):
+ mapping = []
+
+ paths = find_component_paths(path=workspace_path, subdir=subdir)
+ pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
+ results = pool.map(generate_component_data, paths)
+
+ for component_fmris, component_name, component_relative_path in results:
+ for component_fmri in component_fmris:
+ mapping.append({'component_name': component_name,
+ 'component_fmri': component_fmri,
+ 'component_path': component_relative_path})
+
+ component_mapping_file = os.path.join(workspace_path, subdir, COMONENT_MAPPING_FILENAME)
+ with open(component_mapping_file, 'w') as f:
+ f.write(json.dumps(mapping, sort_keys=True, indent=4))
+
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
+ parser.add_argument('--subdir', default='components', help='Directory holding components')
+
+ args = parser.parse_args()
+
+ workspace = args.workspace
+ subdir = args.subdir
+
+ generate_userland_mapping(workspace_path=workspace, subdir=subdir)
+
+
+if __name__ == '__main__':
+ main()
--
Gitblit v1.9.3