From 5d8bcb58722b250c296fc0324f9d06470fb3d7d0 Mon Sep 17 00:00:00 2001
From: Marcel Telka <marcel@telka.sk>
Date: Mon, 08 Apr 2024 06:38:33 +0200
Subject: [PATCH] python/jaraco.context: update to 5.3.0
---
tools/bass-o-matic | 326 ++++++++++++++++++++++++++++++++++--------------------
1 files changed, 206 insertions(+), 120 deletions(-)
diff --git a/tools/bass-o-matic b/tools/bass-o-matic
index dcefdf8..6128b46 100755
--- a/tools/bass-o-matic
+++ b/tools/bass-o-matic
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.6
+#!/usr/bin/python3.9
#
# CDDL HEADER START
#
@@ -30,157 +30,243 @@
import os
import sys
import re
+import glob
import subprocess
+import argparse
+import logging
+import json
+import multiprocessing
+
+try:
+ from scandir import walk
+except ImportError:
+ from os import walk
+
+from bass.component import Component
+
+logger = logging.getLogger('bass-o-matic')
# Locate SCM directories containing Userland components by searching from
# from a supplied top of tree for .p5m files. Once a .p5m file is located,
# that directory is added to the list and no children are searched.
-def FindComponentPaths(path, debug=None):
- expression = re.compile(".+\.p5m$", re.IGNORECASE)
+def FindComponentPaths(path, debug=False, subdir='components',
+ incremental=False, begin_commit=None, end_commit=None):
+ expression = re.compile(r'.+\.p5m$', re.IGNORECASE)
paths = []
if debug:
- print >>debug, "searching %s for component directories" % path
+ logger.debug('searching %s for component directories', path)
- for dirpath, dirnames, filenames in os.walk(path + '/components'):
- found = 0
+ workspace_path = os.path.join(path, subdir)
- for name in filenames:
- if expression.match(name):
- if debug:
- print >>debug, "found %s" % dirpath
- paths.append(dirpath)
- del dirnames[:]
- break
+ if incremental:
+ cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:',
+ '..'.join([begin_commit, end_commit])]
+
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=workspace_path,
+ universal_newlines=True
+ )
+ stdout, stderr = proc.communicate()
+ if debug:
+ if proc.returncode != 0:
+ logger.debug('exit: %d, %s', proc.returncode, stderr)
+
+ for line in stdout.splitlines():
+ line = line.rstrip()
+ # git output might contain empty lines, so we skip them.
+ if not line:
+ continue
+
+ # Every time component is added, modified or moved, Makefile has to be
+ # present. However, this does not yet guarantee that the line is a
+ # real component.
+ filename = os.path.basename(line)
+ dirname = os.path.dirname(line).rsplit(subdir + '/')[-1]
+
+ if filename == 'Makefile':
+ if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
+ not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
+ paths.append(dirname)
+
+ # Some components are using SCM checkout as a source code download method and
+ # COMPONENT_REVISION is not bumped. With this, we will never rebuild them.
+ # In order to rebuild them, we will look for such components and build them
+ # every run. These components are located in openindiana category and we care
+ # only about that category. One exception to this rule is meta-packages/history
+ # component, which holds obsoleted components. We add it to paths manually for
+ # that reason.
+ cmd = ['git', 'grep', '-l', 'GIT_REPO *=']
+
+ proc = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=workspace_path,
+ universal_newlines=True
+ )
+
+ stdout, stderr = proc.communicate()
+ if debug:
+ if proc.returncode != 0:
+ logger.debug('exit: %d, %s', proc.returncode, stderr)
+
+ for line in stdout.splitlines():
+ line = line.rstrip()
+
+ # Only 'openindiana' category.
+ category = line.split('/')[0]
+ if category == 'openindiana':
+ continue
+
+ filename = os.path.basename(line)
+ dirname = os.path.dirname(line)
+
+ if filename == 'Makefile':
+ if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
+ not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
+ paths.append(os.path.dirname(line))
+
+ # Add meta-packages/history only if we build the main repository, where
+ # subdir is equal to 'components'.
+ if subdir == 'components':
+ paths.append('meta-packages/history')
+ # Add encumbered/meta-packages/history only if we build the encumbered repository
+ if subdir == 'components/encumbered':
+ paths.append('encumbered/meta-packages/history')
+
+ paths = list(set(paths))
+
+ else:
+ for dirpath, dirnames, filenames in walk(workspace_path):
+ for name in filenames:
+ if expression.match(name):
+ if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')):
+ if debug:
+ logger.debug('found %s', dirpath)
+ paths.append(dirpath)
+ del dirnames[:]
+ break
return sorted(paths)
-class BassComponent:
- def __init__(self, path=None, debug=None):
- self.debug = debug
- self.path = path
- if path:
- # get supplied packages (cd path ; gmake print-package-names)
- self.supplied_packages = self.run_make(path, 'print-package-names')
-
- # get supplied paths (cd path ; gmake print-package-paths)
- self.supplied_paths = self.run_make(path, 'print-package-paths')
-
- # get required paths (cd path ; gmake print-required-paths)
- self.required_paths = self.run_make(path, 'print-required-paths')
-
- def required(self, component):
- result = False
-
- s1 = set(self.required_paths)
- s2 = set(component.supplied_paths)
- if s1.intersection(s2):
- result = True
-
- return result
-
- def run_make(self, path, targets):
-
- result = list()
-
- if self.debug:
- print >>self.debug, "Executing 'gmake %s' in %s" % (targets, path)
-
- proc = subprocess.Popen(['gmake', targets], cwd=path,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- p = proc.stdout
-
- for out in p:
- result.append(out)
-
- if self.debug:
- proc.wait()
- if proc.returncode != 0:
- print >>self.debug, "exit: %d, %s" % (proc.returncode, proc.stderr.read())
-
- return result
-
- def __str__(self):
- result = "Component:\n\tPath: %s\n" % self.path
- result = result + "\tProvides Package(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_packages)
- result = result + "\tProvides Path(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_paths)
- result = result + "\tRequired Path(s):\n\t\t%s\n" % '\t\t'.join(self.required_paths)
-
- return result
-
-def usage():
- print "Usage: %s [-c|--components=(path|depend)] [-z|--zone (zone)]" % (sys.argv[0].split('/')[-1])
- sys.exit(1)
def main():
- import getopt
- import sys
-
- # FLUSH STDOUT
- sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+ sys.stdout.flush()
components = {}
- debug=None
- components_arg=None
- make_arg=None
- component_arg=None
- template_zone=None
- workspace = os.getenv('WS_TOP')
- try:
- opts, args = getopt.getopt(sys.argv[1:], "w:c:d",
- [ "debug", "workspace=", "components=",
- "make", "component=", "template-zone=" ])
- except getopt.GetoptError, err:
- print str(err)
- usage()
+ COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs']
+ COMPONENTS_ALLOWED_FMRIS = ['fmris']
+ COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies']
+ COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES
- for opt, arg in opts:
- if opt in [ "-w", "--workspace" ]:
- workspace = arg
- elif opt in [ "-l", "--components" ]:
- components_arg = arg
- elif opt in [ "--make" ]:
- make_arg = True
- elif opt in [ "--component" ]:
- component_arg = arg
- elif opt in [ "--template-zone" ]:
- template_zone = arg
- elif opt in [ "-d", "--debug" ]:
- debug = sys.stdout
- else:
- assert False, "unknown option"
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
+ parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS)
+ parser.add_argument('--make', help='Makefile target to invoke')
+ parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata')
+ parser.add_argument('--subdir', default='components', help='Directory holding components')
+ parser.add_argument('-d', '--debug', action='store_true', default=False)
+ parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1'))
+ parser.add_argument('--end-commit', default='HEAD')
- component_paths = FindComponentPaths(workspace, debug)
+ args = parser.parse_args()
+
+ workspace = args.workspace
+ components_arg = args.components
+ make_arg = args.make
+ pkg5_arg = args.pkg5
+ subdir = args.subdir
+ begin_commit = args.begin_commit
+ end_commit = args.end_commit
+ debug = args.debug
+ log_level = logging.WARNING
+
+ if debug:
+ log_level = logging.DEBUG
+
+ logging.basicConfig(level=log_level,
+ format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',)
if make_arg:
- if template_zone:
- print "using template zone %s to create a build environment for %s to run '%s'" % (template_zone, component_arg, ['gmake'] + args)
- proc = subprocess.Popen(['gmake'] + args)
- proc.wait()
+ MAKE=os.getenv("MAKE","gmake")
+
+ # https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
+ JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS",""))
+ if JOBFLAGS:
+ JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) )
+ JOBFLAGS=[JOBFLAGS.group(1)]
+ else:
+ JOBFDS=()
+ JOBFLAGS=[]
+ proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS)
+ rc = proc.wait()
+ sys.exit(rc)
+
+ if pkg5_arg:
+ component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
+ # the component may not be built directly but as a dependency of another component
+ if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')):
+ sys.exit(0)
+ component_pkg5 = os.path.join( os.getcwd(), 'pkg5')
+ if os.path.isfile(component_pkg5):
+ os.remove(component_pkg5)
+ Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0])
sys.exit(0)
+ incremental = False
+ if os.getenv('BASS_O_MATIC_MODE') == 'incremental':
+ incremental = True
+
+ if incremental:
+ component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir,
+ incremental=incremental, begin_commit=begin_commit, end_commit=end_commit)
+ else:
+ component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir)
+
if components_arg:
- if components_arg in [ 'path', 'paths', 'dir', 'dirs', 'directories' ]:
+ if components_arg in COMPONENTS_ALLOWED_PATHS:
for path in component_paths:
- print "%s" % path
-
- elif components_arg in [ 'depend', 'dependencies' ]:
- for path in component_paths:
- components[path] = BassComponent(path, debug)
+ print('{0}'.format(path))
- for c_path in components.keys():
- component = components[c_path]
+ elif components_arg in COMPONENTS_ALLOWED_FMRIS:
+ pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
+ components = pool.map(Component, component_paths)
- for d_path in components.keys():
- if (c_path != d_path and
- component.required(components[d_path])):
- print "%s: %s" % (c_path, d_path)
+ for component in components:
+ for fmri in component.supplied_packages:
+ print('{0}'.format(fmri))
+ elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES:
+ dependencies = {}
+
+ pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
+ components = pool.map(Component, component_paths)
+
+ with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f:
+ data = json.loads(f.read())
+ component_path = {}
+ for entry in data:
+ component_path[entry['fmri']] = entry['path']
+
+ for component in components:
+ selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
+ # Some packages from OpenSolaris only exist in binary form in the pkg repository
+ paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages])
+ # Remove self from the set of paths it depends on
+ paths.discard(selfpath)
+ dependencies[selfpath] = sorted(list(paths))
+
+ dependencies_file = os.path.join(workspace, subdir, 'dependencies.json')
+ with open(dependencies_file, 'w') as f:
+ f.write(json.dumps(dependencies, sort_keys=True, indent=4))
sys.exit(0)
sys.exit(1)
-if __name__ == "__main__":
+
+if __name__ == '__main__':
main()
--
Gitblit v1.9.3