bass-o-matic: fix proc.wait() usage, factor implementation and add pkg5 fil...

Aurelien Larcher

2020-06-01 1e2fed24f6ea2260055ff4be5afe5f8d3709a054

bass-o-matic: fix proc.wait() usage, factor implementation and add pkg5 file generation

 tools/bass-o-matic

@@ -42,6 +42,8 @@
except ImportError:
    from os import walk

from bass.component import BassComponent

logger = logging.getLogger('bass-o-matic')

# Locate SCM directories containing Userland components by searching from
@@ -148,57 +150,6 @@
    return sorted(paths)


class BassComponent(object):
    def __init__(self, path=None, debug=False):
        self.debug = debug
        self.path = path
        if path:
            # get supplied packages    (cd path ; gmake print-package-names)
            self.supplied_packages = self.run_make(path, 'print-package-names')

            # get dependencies
            self.required_packages = self.run_make(path, 'print-required-packages')

    def required(self, component):
        result = False

        s1 = set(self.required_packages)
        s2 = set(component.supplied_packages)
        if s1.intersection(s2):
            result = True

        return result

    def run_make(self, path, targets):

        result = []

        if self.debug:
            logger.debug('Executing \'gmake %s\' in %s', targets, path)

        proc = subprocess.Popen(['gmake', '-s', targets],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                cwd=path,
                                universal_newlines=True)
        for out in proc.stdout:
            result.append(out.rstrip())

        proc.wait()
        if self.debug:
            if proc.returncode != 0:
                logger.debug('exit: %d, %s', proc.returncode, proc.stderr.read())

        return result

    def __str__(self):
        result = 'Component:\n\tPath: %s\n' % self.path
        result += '\tProvides Package(s):\n\t\t%s\n' % '\t\t'.join(self.supplied_packages)
        result += '\tRequired Package(s):\n\t\t%s\n' % '\t\t'.join(self.required_packages)

        return result


def main():
    sys.stdout.flush()

@@ -213,6 +164,7 @@
    parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
    parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS)
    parser.add_argument('--make', help='Makefile target to invoke')
    parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata')
    parser.add_argument('--subdir', default='components', help='Directory holding components')
    parser.add_argument('-d', '--debug', action='store_true', default=False)
    parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1'))
@@ -222,8 +174,9 @@

    workspace = args.workspace
    components_arg = args.components
    subdir = args.subdir
    make_arg = args.make
    pkg5_arg = args.pkg5
    subdir = args.subdir
    begin_commit = args.begin_commit
    end_commit = args.end_commit
    debug = args.debug
@@ -239,6 +192,14 @@
        proc = subprocess.Popen(['gmake'] + [make_arg])
        rc = proc.wait()
        sys.exit(rc)

    if pkg5_arg:
        component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
        component_pkg5 = os.path.join( os.getcwd(), 'pkg5')
        if os.path.isfile(component_pkg5):
            os.remove(component_pkg5)
        BassComponent(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0])
        sys.exit(0)

    incremental = False
    if os.getenv('BASS_O_MATIC_MODE') == 'incremental':
@@ -269,9 +230,16 @@
            pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
            components = pool.map(BassComponent, component_paths)

            with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f:
                     data = json.loads(f.read())
            component_path = {}
            for entry in data:
                component_path[entry['fmri']] = entry['path']

            for component in components:
                for fmri in component.supplied_packages:
                    dependencies[fmri] = component.required_packages
                    paths = set([component_path[i] for i in component.required_packages])
                    dependencies[fmri] = list(paths)

            dependencies_file = os.path.join(workspace, subdir, 'dependencies.json')
            with open(dependencies_file, 'w') as f:

 tools/bass/component.py

New file
@@ -0,0 +1,109 @@
#!/usr/bin/python3.5
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright (c) 2010, Oracle and/or it's affiliates.  All rights reserved.
#
#
# bass-o-matic.py
#  A simple program to enumerate components in the userland gate and report
#  on dependency related information.
#

import os
import subprocess
import json

class BassComponent(object):
    def __init__(self, path=None, debug=False):
        self.debug = debug
        self.path = path
        if path:
            component_pkg5_file = os.path.join(self.path, 'pkg5')
            if not os.path.isfile(component_pkg5_file):
                # get component name
                self.component_name = self.run_make(path, 'print-value-COMPONENT_NAME')[0]
                # get supplied packages, this may trigger the creation of a pkg5.fmris file
                self.supplied_packages = self.run_make(path, 'print-package-names')
                # always update fmris if list is overriden
                component_pkg5_fmris_file = os.path.join(self.path, 'pkg5.fmris')
                if os.path.isfile(component_pkg5_fmris_file):
                    with open(component_pkg5_fmris_file, 'r') as f:
                        self.supplied_packages = f.read().splitlines()

                # get dependencies
                self.required_packages = self.run_make(path, 'print-required-packages')

                data = { 'name' : self.component_name,
                         'fmris': self.supplied_packages,
                         'dependencies' : self.required_packages }
                with open(component_pkg5_file, 'w') as f:
                    f.write(json.dumps(data, sort_keys=True, indent=4))
            else:
                with open(component_pkg5_file, 'r') as f:
                    data = json.loads(f.read())
                if not data:
                    raise ValueError('Component pkg5 data is empty for path ' + self.path + '.')
                self.component_name = data['name']
                self.supplied_packages = data['fmris']
                self.required_packages = data['dependencies']
            if not self.supplied_packages or not self.supplied_packages[0]:
                raise ValueError('Empty list of supplied FMRIs\npath = ' + self.path)

    def required(self, component):
        result = False

        s1 = set(self.required_packages)
        s2 = set(component.supplied_packages)
        if s1.intersection(s2):
            result = True

        return result

    def run_make(self, path, targets):

        result = []

        if self.debug:
            logger.debug('Executing \'gmake %s\' in %s', targets, path)

        proc = subprocess.Popen(['gmake', '-s', targets],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                cwd=path,
                                universal_newlines=True)
        stdout, stderr = proc.communicate()

        for out in stdout.splitlines():
            result.append(out.rstrip())

        if self.debug:
            if proc.returncode != 0:
                logger.debug('exit: %d, %s', proc.returncode, stderr)

        return result

    def __str__(self):
        result = 'Component:\n\tPath: %s\n' % self.path
        result += '\tProvides Package(s):\n\t\t%s\n' % '\t\t'.join(self.supplied_packages)
        result += '\tRequired Package(s):\n\t\t%s\n' % '\t\t'.join(self.required_packages)

        return result


 tools/userland-mapping

@@ -28,6 +28,8 @@
import subprocess
import multiprocessing

from bass.component import BassComponent

try:
    from scandir import walk
except ImportError:
@@ -35,7 +37,7 @@

logger = logging.getLogger('userland-mapping')

COMONENT_MAPPING_FILENAME = 'mapping.json'
COMPONENT_MAPPING_FILENAME = 'mapping.json'


def find_component_paths(path, subdir='components', debug=False):
@@ -56,18 +58,14 @@

def generate_component_data(component_path, subdir='components'):
    result = []
    component = BassComponent(path=component_path)
    component_name = component.component_name
    if not component_name:
        raise ValueError('Component name is empty for path ' + component_path + '.')
    component_fmris = component.supplied_packages
    if not component_fmris:
        raise ValueError('Component FMRIs is empty for path ' + component_path + '.')

    proc = subprocess.Popen(['gmake', '-s', 'print-value-COMPONENT_NAME', 'print-package-names'],
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            cwd=component_path,
                            universal_newlines=True)

    for out in proc.stdout:
        result.append(out.rstrip())

    component_name = result[0]
    component_fmris = result[1:]
    component_relative_path = component_path.split(os.path.join(os.environ['WS_TOP'], subdir))[-1].replace('/', '', 1)

    return component_fmris, component_name, component_relative_path
@@ -82,11 +80,11 @@

    for component_fmris, component_name, component_relative_path in results:
        for component_fmri in component_fmris:
            mapping.append({'component_name': component_name,
                            'component_fmri': component_fmri,
                            'component_path': component_relative_path})
            mapping.append({'name': component_name,
                            'fmri': component_fmri,
                            'path': component_relative_path})

    component_mapping_file = os.path.join(workspace_path, subdir, COMONENT_MAPPING_FILENAME)
    component_mapping_file = os.path.join(workspace_path, subdir, COMPONENT_MAPPING_FILENAME)
    with open(component_mapping_file, 'w') as f:
        f.write(json.dumps(mapping, sort_keys=True, indent=4))

@@ -103,7 +101,6 @@
    subdir = args.subdir

    generate_userland_mapping(workspace_path=workspace, subdir=subdir)


if __name__ == '__main__':
    main()