| | |
| | | #!/usr/bin/python2.6 |
| | | #!/usr/bin/python3.9 |
| | | # |
| | | # CDDL HEADER START |
| | | # |
| | |
| | | import os |
| | | import sys |
| | | import re |
| | | import glob |
| | | import subprocess |
| | | import argparse |
| | | import logging |
| | | import json |
| | | import multiprocessing |
| | | |
| | | try: |
| | | from scandir import walk |
| | | except ImportError: |
| | | from os import walk |
| | | |
| | | from bass.component import Component |
| | | |
| | | logger = logging.getLogger('bass-o-matic') |
| | | |
| | | # Locate SCM directories containing Userland components by searching from |
| | | # from a supplied top of tree for .p5m files. Once a .p5m file is located, |
| | | # that directory is added to the list and no children are searched. |
| | | def FindComponentPaths(path, debug=None): |
| | | expression = re.compile(".+\.p5m$", re.IGNORECASE) |
| | | def FindComponentPaths(path, debug=False, subdir='components', |
| | | incremental=False, begin_commit=None, end_commit=None): |
| | | expression = re.compile(r'.+\.p5m$', re.IGNORECASE) |
| | | |
| | | paths = [] |
| | | |
| | | if debug: |
| | | print >>debug, "searching %s for component directories" % path |
| | | logger.debug('searching %s for component directories', path) |
| | | |
| | | for dirpath, dirnames, filenames in os.walk(path + '/components'): |
| | | found = 0 |
| | | workspace_path = os.path.join(path, subdir) |
| | | |
| | | for name in filenames: |
| | | if expression.match(name): |
| | | if debug: |
| | | print >>debug, "found %s" % dirpath |
| | | paths.append(dirpath) |
| | | del dirnames[:] |
| | | break |
| | | if incremental: |
| | | cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:', |
| | | '..'.join([begin_commit, end_commit])] |
| | | |
| | | proc = subprocess.Popen(cmd, |
| | | stdout=subprocess.PIPE, |
| | | stderr=subprocess.PIPE, |
| | | cwd=workspace_path, |
| | | universal_newlines=True |
| | | ) |
| | | stdout, stderr = proc.communicate() |
| | | if debug: |
| | | if proc.returncode != 0: |
| | | logger.debug('exit: %d, %s', proc.returncode, stderr) |
| | | |
| | | for line in stdout.splitlines(): |
| | | line = line.rstrip() |
| | | # git output might contain empty lines, so we skip them. |
| | | if not line: |
| | | continue |
| | | |
| | | # Every time component is added, modified or moved, Makefile has to be |
| | | # present. However, this does not yet guarantee that the line is a |
| | | # real component. |
| | | filename = os.path.basename(line) |
| | | dirname = os.path.dirname(line).rsplit(subdir + '/')[-1] |
| | | |
| | | if filename == 'Makefile': |
| | | if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ |
| | | not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): |
| | | paths.append(dirname) |
| | | |
| | | # Some components are using SCM checkout as a source code download method and |
| | | # COMPONENT_REVISION is not bumped. With this, we will never rebuild them. |
| | | # In order to rebuild them, we will look for such components and build them |
| | | # every run. These components are located in openindiana category and we care |
| | | # only about that category. One exception to this rule is meta-packages/history |
| | | # component, which holds obsoleted components. We add it to paths manually for |
| | | # that reason. |
| | | cmd = ['git', 'grep', '-l', 'GIT_REPO *='] |
| | | |
| | | proc = subprocess.Popen(cmd, |
| | | stdout=subprocess.PIPE, |
| | | stderr=subprocess.PIPE, |
| | | cwd=workspace_path, |
| | | universal_newlines=True |
| | | ) |
| | | |
| | | stdout, stderr = proc.communicate() |
| | | if debug: |
| | | if proc.returncode != 0: |
| | | logger.debug('exit: %d, %s', proc.returncode, stderr) |
| | | |
| | | for line in stdout.splitlines(): |
| | | line = line.rstrip() |
| | | |
| | | # Only 'openindiana' category. |
| | | category = line.split('/')[0] |
| | | if category == 'openindiana': |
| | | continue |
| | | |
| | | filename = os.path.basename(line) |
| | | dirname = os.path.dirname(line) |
| | | |
| | | if filename == 'Makefile': |
| | | if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ |
| | | not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): |
| | | paths.append(os.path.dirname(line)) |
| | | |
| | | # Add meta-packages/history only if we build the main repository, where |
| | | # subdir is equal to 'components'. |
| | | if subdir == 'components': |
| | | paths.append('meta-packages/history') |
| | | # Add encumbered/meta-packages/history only if we build the encumbered repository |
| | | if subdir == 'components/encumbered': |
| | | paths.append('encumbered/meta-packages/history') |
| | | |
| | | paths = list(set(paths)) |
| | | |
| | | else: |
| | | for dirpath, dirnames, filenames in walk(workspace_path): |
| | | for name in filenames: |
| | | if expression.match(name): |
| | | if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')): |
| | | if debug: |
| | | logger.debug('found %s', dirpath) |
| | | paths.append(dirpath) |
| | | del dirnames[:] |
| | | break |
| | | |
| | | return sorted(paths) |
| | | |
| | | class BassComponent: |
| | | def __init__(self, path=None, debug=None): |
| | | self.debug = debug |
| | | self.path = path |
| | | if path: |
| | | # get supplied packages (cd path ; gmake print-package-names) |
| | | self.supplied_packages = self.run_make(path, 'print-package-names') |
| | | |
| | | # get supplied paths (cd path ; gmake print-package-paths) |
| | | self.supplied_paths = self.run_make(path, 'print-package-paths') |
| | | |
| | | # get required paths (cd path ; gmake print-required-paths) |
| | | self.required_paths = self.run_make(path, 'print-required-paths') |
| | | |
| | | def required(self, component): |
| | | result = False |
| | | |
| | | s1 = set(self.required_paths) |
| | | s2 = set(component.supplied_paths) |
| | | if s1.intersection(s2): |
| | | result = True |
| | | |
| | | return result |
| | | |
| | | def run_make(self, path, targets): |
| | | |
| | | result = list() |
| | | |
| | | if self.debug: |
| | | print >>self.debug, "Executing 'gmake %s' in %s" % (targets, path) |
| | | |
| | | proc = subprocess.Popen(['gmake', targets], cwd=path, |
| | | stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| | | p = proc.stdout |
| | | |
| | | for out in p: |
| | | result.append(out) |
| | | |
| | | if self.debug: |
| | | proc.wait() |
| | | if proc.returncode != 0: |
| | | print >>self.debug, "exit: %d, %s" % (proc.returncode, proc.stderr.read()) |
| | | |
| | | return result |
| | | |
| | | def __str__(self): |
| | | result = "Component:\n\tPath: %s\n" % self.path |
| | | result = result + "\tProvides Package(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_packages) |
| | | result = result + "\tProvides Path(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_paths) |
| | | result = result + "\tRequired Path(s):\n\t\t%s\n" % '\t\t'.join(self.required_paths) |
| | | |
| | | return result |
| | | |
| | | def usage(): |
| | | print "Usage: %s [-c|--components=(path|depend)] [-z|--zone (zone)]" % (sys.argv[0].split('/')[-1]) |
| | | sys.exit(1) |
| | | |
| | | def main(): |
| | | import getopt |
| | | import sys |
| | | |
| | | # FLUSH STDOUT |
| | | sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
| | | sys.stdout.flush() |
| | | |
| | | components = {} |
| | | debug=None |
| | | components_arg=None |
| | | make_arg=None |
| | | component_arg=None |
| | | template_zone=None |
| | | workspace = os.getenv('WS_TOP') |
| | | |
| | | try: |
| | | opts, args = getopt.getopt(sys.argv[1:], "w:c:d", |
| | | [ "debug", "workspace=", "components=", |
| | | "make", "component=", "template-zone=" ]) |
| | | except getopt.GetoptError, err: |
| | | print str(err) |
| | | usage() |
| | | COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs'] |
| | | COMPONENTS_ALLOWED_FMRIS = ['fmris'] |
| | | COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies'] |
| | | COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES |
| | | |
| | | for opt, arg in opts: |
| | | if opt in [ "-w", "--workspace" ]: |
| | | workspace = arg |
| | | elif opt in [ "-l", "--components" ]: |
| | | components_arg = arg |
| | | elif opt in [ "--make" ]: |
| | | make_arg = True |
| | | elif opt in [ "--component" ]: |
| | | component_arg = arg |
| | | elif opt in [ "--template-zone" ]: |
| | | template_zone = arg |
| | | elif opt in [ "-d", "--debug" ]: |
| | | debug = sys.stdout |
| | | else: |
| | | assert False, "unknown option" |
| | | parser = argparse.ArgumentParser() |
| | | parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace') |
| | | parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS) |
| | | parser.add_argument('--make', help='Makefile target to invoke') |
| | | parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata') |
| | | parser.add_argument('--subdir', default='components', help='Directory holding components') |
| | | parser.add_argument('-d', '--debug', action='store_true', default=False) |
| | | parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1')) |
| | | parser.add_argument('--end-commit', default='HEAD') |
| | | |
| | | component_paths = FindComponentPaths(workspace, debug) |
| | | args = parser.parse_args() |
| | | |
| | | workspace = args.workspace |
| | | components_arg = args.components |
| | | make_arg = args.make |
| | | pkg5_arg = args.pkg5 |
| | | subdir = args.subdir |
| | | begin_commit = args.begin_commit |
| | | end_commit = args.end_commit |
| | | debug = args.debug |
| | | log_level = logging.WARNING |
| | | |
| | | if debug: |
| | | log_level = logging.DEBUG |
| | | |
| | | logging.basicConfig(level=log_level, |
| | | format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',) |
| | | |
| | | if make_arg: |
| | | if template_zone: |
| | | print "using template zone %s to create a build environment for %s to run '%s'" % (template_zone, component_arg, ['gmake'] + args) |
| | | proc = subprocess.Popen(['gmake'] + args) |
| | | rc = proc.wait() |
| | | MAKE=os.getenv("MAKE","gmake") |
| | | |
| | | # https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html |
| | | JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS","")) |
| | | if JOBFLAGS: |
| | | JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) ) |
| | | JOBFLAGS=[JOBFLAGS.group(1)] |
| | | else: |
| | | JOBFDS=() |
| | | JOBFLAGS=[] |
| | | proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS) |
| | | rc = proc.wait() |
| | | sys.exit(rc) |
| | | |
| | | if pkg5_arg: |
| | | component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) |
| | | # the component may not be built directly but as a dependency of another component |
| | | if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')): |
| | | sys.exit(0) |
| | | component_pkg5 = os.path.join( os.getcwd(), 'pkg5') |
| | | if os.path.isfile(component_pkg5): |
| | | os.remove(component_pkg5) |
| | | Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0]) |
| | | sys.exit(0) |
| | | |
| | | incremental = False |
| | | if os.getenv('BASS_O_MATIC_MODE') == 'incremental': |
| | | incremental = True |
| | | |
| | | if incremental: |
| | | component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir, |
| | | incremental=incremental, begin_commit=begin_commit, end_commit=end_commit) |
| | | else: |
| | | component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir) |
| | | |
| | | if components_arg: |
| | | if components_arg in [ 'path', 'paths', 'dir', 'dirs', 'directories' ]: |
| | | if components_arg in COMPONENTS_ALLOWED_PATHS: |
| | | for path in component_paths: |
| | | print "%s" % path |
| | | |
| | | elif components_arg in [ 'depend', 'dependencies' ]: |
| | | for path in component_paths: |
| | | components[path] = BassComponent(path, debug) |
| | | print('{0}'.format(path)) |
| | | |
| | | for c_path in components.keys(): |
| | | component = components[c_path] |
| | | elif components_arg in COMPONENTS_ALLOWED_FMRIS: |
| | | pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) |
| | | components = pool.map(Component, component_paths) |
| | | |
| | | for d_path in components.keys(): |
| | | if (c_path != d_path and |
| | | component.required(components[d_path])): |
| | | print "%s: %s" % (c_path, d_path) |
| | | for component in components: |
| | | for fmri in component.supplied_packages: |
| | | print('{0}'.format(fmri)) |
| | | |
| | | elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES: |
| | | dependencies = {} |
| | | |
| | | pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) |
| | | components = pool.map(Component, component_paths) |
| | | |
| | | with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f: |
| | | data = json.loads(f.read()) |
| | | component_path = {} |
| | | for entry in data: |
| | | component_path[entry['fmri']] = entry['path'] |
| | | |
| | | for component in components: |
| | | selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) |
| | | # Some packages from OpenSolaris only exist in binary form in the pkg repository |
| | | paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages]) |
| | | # Remove self from the set of paths it depends on |
| | | paths.discard(selfpath) |
| | | dependencies[selfpath] = sorted(list(paths)) |
| | | |
| | | dependencies_file = os.path.join(workspace, subdir, 'dependencies.json') |
| | | with open(dependencies_file, 'w') as f: |
| | | f.write(json.dumps(dependencies, sort_keys=True, indent=4)) |
| | | sys.exit(0) |
| | | |
| | | sys.exit(1) |
| | | |
| | | if __name__ == "__main__": |
| | | |
| | | if __name__ == '__main__': |
| | | main() |