Add userland incorporation Major differences from upstream: - we can const...

Alexander Pyhalov

2015-10-03 10a7de4ae7aaa21ec3d5ed5151cbac572d878f01

Add userland incorporation
Major differences from upstream:
	- we can construct only one incorporation
	- incorporation should be generated explicitly (publish doesn't depend on it)
	- packages shouldn't explicitly ask to be incorporated, but can ask to be not incorporated
	- we include timestamps in incorporation, as not all packages are properly versioned

 components/Makefile

@@ -129,17 +129,18 @@
     $(BASS_O_MATIC) $(TEMPLATE_ZONE:%=--template-zone %) \
             $(@:%=--component %) --make $(TARGET) $(LOG)

incorporation:
    $(WS_TOOLS)/userland-incorporator --repository $(PKG_REPO) \
      --version=0.$(OS_VERSION),$(BUILD_VERSION) -p $(PUBLISHER) \
      -c $(CONSOLIDATION) --destdir=$(WS_MACH) $(INCORPORATE_WERROR)
    for incorporation in $(WS_MACH)/*.p5m ; do \
        $(PKGSEND) -s $(PKG_REPO) publish --fmri-in-manifest \
                   $${incorporation} ; \
    done


publish:
    $(PKGREPO) rebuild -s $(PKG_REPO)
# No need for userland incorporation on OI
#    $(WS_TOOLS)/userland-incorporator --repository $(PKG_REPO) \
#      -p pkg:/consolidation/$(CONSOLIDATION)/$(CONSOLIDATION)-incorporation@0.$(OS_VERSION),$(BUILD_VERSION) \
#      -s "$(CONSOLIDATION) consolidation incorporation" \
#      -d "This incorporation constrains packages from the $(CONSOLIDATION) consolidation" \
#      -c $(CONSOLIDATION) >$(WS_LOGS)/$(CONSOLIDATION)-incorporation.p5m
#    $(PKGSEND) -s $(PKG_REPO) publish --fmri-in-manifest \
#      $(WS_LOGS)/$(CONSOLIDATION)-incorporation.p5m
# pkglint all of the published manifests in one batch.
    $(PKGREPO) refresh -s $(PKG_REPO)
ifdef CANONICAL_REPO
    @echo 'pkglinting all package manifests...'
    @$(ENV) PYTHONPATH=$(WS_TOOLS)/python $(PKGLINT) \

 components/meta-packages/entire-incorporation/entire.p5m

@@ -17,3 +17,5 @@
set name=pkg.description value="incorporation to lock all system packages to same build (empty package)"
set name=pkg.summary value="incorporation to lock all system packages to same build (empty package)"
set name=org.opensolaris.consolidation value=$(CONSOLIDATION)

depend type=require fmri=consolidation/userland/userland-incorporation

 make-rules/shared-macros.mk

@@ -46,13 +46,15 @@
    $(shell hg root 2>/dev/null || git rev-parse --show-toplevel)
endif

WS_LOGS =    $(WS_TOP)/$(MACH)/logs
WS_REPO =    $(WS_TOP)/$(MACH)/repo
WS_TOOLS =    $(WS_TOP)/tools
WS_MAKE_RULES =    $(WS_TOP)/make-rules
WS_COMPONENTS =    $(WS_TOP)/components
WS_INCORPORATIONS =    $(WS_TOP)/incorporations
WS_LINT_CACHE =    $(WS_TOP)/$(MACH)/pkglint-cache
WS_MACH =       $(WS_TOP)/$(MACH)
WS_LOGS =       $(WS_MACH)/logs
WS_REPO =       $(WS_MACH)/repo
WS_TOOLS =      $(WS_TOP)/tools
WS_MAKE_RULES = $(WS_TOP)/make-rules
WS_COMPONENTS = $(WS_TOP)/components
WS_LICENSES =   $(WS_TOP)/licenses
WS_INCORPORATIONS =     $(WS_TOP)/incorporations
WS_LINT_CACHE = $(WS_MACH)/pkglint-cache

# we want our pkg piplines to fail if there is an error
# (like if pkgdepend fails in the middle of a pipe), but

 tools/userland-incorporator

@@ -1,4 +1,4 @@
#!/usr/bin/perl
#!/usr/bin/python2.6
#
# CDDL HEADER START
#
@@ -19,78 +19,155 @@
#
# CDDL HEADER END
#
# Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
#
#
# incorporator - an utility to incorporate packages in a repo
#

use Getopt::Long;
import subprocess
import json
import sys
import getopt
import re
import os.path

sub enumerate_packages {
    local ($repository, $publisher, @fmris) = @_;
        my @packages = ();
Werror = False    # set to true to exit with any warning

    #printf "/usr/bin/pkg list -ng $repository @fmris\n";
    open($fp, "-|", "/usr/bin/pkgrepo", "list", "-H", "-s", $repository,
          "-p", $publisher, @fmris) ||
                  die "pkg: $!";
    while (<$fp>) {
def warning(msg):
    if Werror == True:
        print >>sys.stderr, "ERROR: %s" % msg
        sys.exit(1)
    else:
        print >>sys.stderr, "WARNING: %s" % msg

        # lines should be in the form:
        #   publisher   package   [r|o] version,branch:timestamp
        if (/^(\S+)\s+(\S+)\s+\S?\s+([\d.]+),[\d.]+-([\d.]+):.+$/) {
            my ($package) = ();
class Incorporation(object):
    name = None
    version = '5.11'
    packages = {}

            $package->{publisher} = $1;
            $package->{name} = $2;
            $package->{version} = $3;
            $package->{branch} = $4;
    def __init__(self, name, version):
        self.name = name
        self.version = version
        self.packages = {}

            push(@packages, $package);
        } else {
            printf STDERR "SKIP: %s", $_;
        }
    }
    def __package_to_str(self, name, version):
        # strip the :timestamp from the version string
        # version = version.split(':', 1)[0]
        # strip the ,{build-release} from the version string
        # version = re.sub(",[\d\.]+", "", version) 

    #printf "returning %s\n", $_->{name} for (@packages);
        return "depend fmri=%s@%s facet.version-lock.%s=true type=incorporate" % (name, version, name)

    return @packages;
}
    def add_package(self, name, version):
        self.packages[name] = version

sub print_incorporate {
    local (%package) = @_;
    my $facet = "facet.version-lock.$package->{name}";
    def __str__(self):
        result = """
set name=pkg.fmri value=pkg:/%s@%s
set name=info.classification value="org.opensolaris.category.2008:Meta Packages/Incorporations"
set name=org.opensolaris.consolidation value=userland
set name=pkg.depend.install-hold value=core-os.userland
set name=pkg.summary value="userland consolidation incorporation (%s)"
set name=pkg.description value="This incorporation constrains packages from the userland consolidation"
""" % (self.name, self.version, self.name)

    printf "depend fmri=%s@%s-%s %s=true type=incorporate\n",
        $package->{name}, $package->{version}, $package->{branch},
        $facet;
}
        names = self.packages.keys()
        names.sort()
        for name in names:
            result += (self.__package_to_str(name, self.packages[name]) + '\n')

my ($repository, $fmri, $summary, $description, $consolidation) = ();

$consolidation = 'userland';

GetOptions("R|repository=s" => \$repository, "v|version=s" => \$version,
       "s|summary=s" => \$summary, "d|description=s" => \$description,
       "p|package=s" => \$fmri, "f|fmri=s" => \@fmris,
       "c|consolidation=s" => \$consolidation);
        return result

#
# print the incorporation
# This should probably use the pkg APIs at some point, but this appears to be
# a stable and less complicated interface to gathering information from the
# manifests in the package repo.
#
printf "set name=pkg.fmri value=%s\n", $fmri;
printf "set name=pkg.summary value='%s'\n", $summary;
printf "set name=pkg.description value='%s'\n", $description;
printf "set name=org.opensolaris.consolidation value=%s\n",
        $consolidation;
printf "set name=pkg.depend.install-hold value=core-os.%s\n",
        $consolidation;
printf "set name=info.classification value='org.opensolaris.category.2008:Meta Packages/Incorporations'\n";
def get_incorporations(repository, publisher, inc_version='5.11'):
    tmp = subprocess.Popen(["/usr/bin/pkgrepo", "list", "-F", "json",
                                                        "-s", repository,
                                                        "-p", publisher],
                           stdout=subprocess.PIPE)
    incorporations = {}
    packages = json.load(tmp.stdout)

@packages = enumerate_packages($repository, $consolidation, @fmris);
for (@packages) {
    printf "depend fmri=pkg:/%s@%s-%s %s=true type=incorporate\n",
        %$_->{name}, %$_->{version}, %$_->{branch},
        "facet.version-lock.".%$_->{name};
}
    # Check for multiple versions of packages in the repo, but keep track of
    # the latest one.
    versions = {}
    for package in packages:
        pkg_name = package['name']
        pkg_version = package['version']

        if pkg_name in versions:
            warning("%s is in the repo at multiple versions (%s, %s)" % (pkg_name, pkg_version, versions[pkg_name]))
            pkg_version = max(pkg_version, versions[pkg_name])
        versions[pkg_name] = pkg_version

    for package in packages:
        pkg_name = package['name']
        pkg_version = package['version']

        # skip older packages and those that explicitly don't want to be incorporated
    if 'pkg.tmp.noincorporate' in package or pkg_version != versions[pkg_name]:
            continue

    # We don't want to support multiple incorporations for now
        # a dict inside a list inside a dict
        # incorporate = package['pkg.tmp.incorporate'][0]['value']
        
        inc_name='consolidation/userland/userland-incorporation'
        # for inc_name in incorporate:
            # if we haven't started to build this incorporation, create one.
        if inc_name not in incorporations:
           incorporations[inc_name] = Incorporation(inc_name, inc_version)
        # find the incorporation and add the package
        tmp = incorporations[inc_name]
        tmp.add_package(pkg_name, pkg_version)
    return incorporations

def main_func():
    global Werror

    try: 
        opts, pargs = getopt.getopt(sys.argv[1:], "c:s:p:v:d:w",
                                    ["repository=", "publisher=", "version=",
                                     "consolidation=", "destdir=", "Werror"])
    except getopt.GetoptError, e:
        usage(_("illegal option: %s") % e.opt)

    repository = None
    publisher = None
    version = None
    destdir = None
    consolidation = None

    for opt, arg in opts:
        if opt in ("-s", "--repository"):
            repository = arg
        elif opt in ("-p", "--publisher"):
            publisher = arg
        elif opt in ("-v", "--version"):
            version = arg
        elif opt in ("-d", "--destdir"):
            destdir = arg
        elif opt in ("-c", "--consolidation"):
            consolidation = arg
        elif opt in ("-w", "--Werror"):
            Werror = True

    incorporations = get_incorporations(repository, publisher, version)

    for incorporation_name in incorporations.keys():
        filename = ''
        if destdir != None:
            filename = destdir + '/'
        filename += os.path.basename(incorporation_name) + '.p5m'

        print("Writing %s manifest to %s" % (incorporation_name, filename))
        fd = open(filename, "w+")
        fd.write(str(incorporations[incorporation_name]))
        fd.close()

if __name__ == "__main__":
    main_func()