fix using/downloading shared archives

Norm Jacobs

2010-11-05 6ddf488f8a457bc6d322eb8653f2e6dc33d659a8

fix using/downloading shared archives

 make-rules/prep.mk

@@ -36,15 +36,15 @@
    $(GPATCH) -d $(@D) $(GPATCH_FLAGS) < $<
    $(TOUCH) $@

$(COMPONENT_ARCHIVE):    Makefile
$(USERLAND_ARCHIVES)$(COMPONENT_ARCHIVE):    Makefile
    $(FETCH) --file $@ \
        $(COMPONENT_ARCHIVE_URL:%=--url %) \
        $(COMPONENT_ARCHIVE_HASH:%=--hash %)
    $(TOUCH) $@

$(COMPONENT_SRC)/.unpacked:    $(COMPONENT_ARCHIVE) Makefile $(PATCHES)
$(COMPONENT_SRC)/.unpacked:    $(USERLAND_ARCHIVES)$(COMPONENT_ARCHIVE) Makefile $(PATCHES)
    $(RM) -r $(COMPONENT_SRC)
    $(UNPACK) $(UNPACK_ARGS) $(COMPONENT_ARCHIVE)
    $(UNPACK) $(UNPACK_ARGS) $(USERLAND_ARCHIVES)$(COMPONENT_ARCHIVE)
    $(TOUCH) $@

$(COMPONENT_SRC)/.patched:    $(COMPONENT_SRC)/.unpacked $(STAMPS)
@@ -56,7 +56,7 @@

prep::    $(COMPONENT_SRC)/.prep

download::    $(COMPONENT_ARCHIVE)
download::    $(USERLAND_ARCHIVES)$(COMPONENT_ARCHIVE)

clean::
    $(RM) -r $(CLEAN_PATHS)

 tools/userland-fetch

@@ -30,6 +30,8 @@

import os
import sys
import shutil
from urllib import splittype, urlopen

def validate(filename, hash):
    import hashlib
@@ -56,133 +58,133 @@

    return m.hexdigest() == value

def download(url, filename = None, hash = None, verbose = False):
    from urllib import splittype, urlopen
    import hashlib

def download(url, filename = None):
    src = None
    filename = None

    #print "Downloading %s from %s" % (filename, url)

    # open the download source
    if splittype(url)[0]:
        try:
            src = urlopen(url)
        except IOError:
            return None
        remote_name = src.geturl().split('/')[-1]
    elif os.path.isfile(url):
        os.symlink(url, filename)
        return filename, filehash(filename)
        src = open(url, 'r')
        remote_name = url.split('/')[-1]
    else:
    try:
        src = urlopen(url)
    except IOError:
        return None

    if filename == None:
        filename = remote_name
        filename = src.geturl().split('/')[-1]

    # if we have a source to download, open a destination and copy the data
    if src:
    try:
        dst = open(filename, 'wb');
        if verbose:
            print "Downloading %s from %s" % (filename, url)
        while True:
            block = src.read()
            if block == '':
                break;
            dst.write(block)
        dst.close()
    except IOError:
        src.close()
        return None

    while True:
        block = src.read()
        if block == '':
            break;
        dst.write(block)

    src.close()
    dst.close()

    # return the name of the file that we downloaded the data to.
    return filename

def download_paths(search, filename):
    tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
    if tmp:
        search += tmp.split(' ')
    base = os.path.basename(filename)

def download_paths(search, filename, url):
    urls = list()

    for path in search:
        urls.append(path+'/'+base)
    if filename != None:
        tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
        if tmp:
            search += tmp.split(' ')

        file = os.path.basename(filename)

        urls = [ base + '/' + file for base in search ]

        # filename should always be first
        if filename in urls:
            urls.remove(filename)
        urls.insert(0, filename)

    # command line url is a fallback, so it's last
    if url != None and url not in urls:
        urls.append(url)

    return urls

def usage():
    print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1])
    print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
    sys.exit(1)

def main():
    from urllib import splittype, urlopen
    import getopt
    import sys

    # FLUSH STDOUT 
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    verbose = False
    filename = None
    url = None
    hash = None
    search = list()
    file_arg = None
    link_arg = False
    hash_arg = None
    url_arg = None
    search_list = list()

    try:
        opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v",
            ["file=", "hash=", "search=", "url=", "verbose"])
        opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
            ["file=", "link", "hash=", "search=", "url="])
    except getopt.GetoptError, err:
        print str(err)
        usage()

    for opt, arg in opts:
        if opt in [ "-f", "--file" ]:
            filename = arg
            file_arg = arg
        elif opt in [ "-l", "--link" ]:
            link_arg = True
        elif opt in [ "-h", "--hash" ]:
            hash = arg
            hash_arg = arg
        elif opt in [ "-s", "--search" ]:
            search.append(arg)
            search_list.append(arg)
        elif opt in [ "-u", "--url" ]:
            url = arg
        elif opt in [ "-v", "--verbose" ]:
            verbose = True
            url_arg = arg
        else:
            assert False, "unknown option"

    if url == None:
    if url_arg == None:
        usage()

    print "Fetching %s" % filename
    for url in download_paths(search_list, file_arg, url_arg):
        print "Source %s..." % url,

    if validate(filename, hash):
        print "\tcurrent copy is ok, nothing to do"
        sys.exit(0)
        scheme, path = splittype(url)
        name = file_arg

    # generate a list of URLS to search for a suitable download
    urls = download_paths(search, filename)
    urls.append(url)

    for url in urls:
        print "\tTrying %s..." % url,

        if os.path.isfile(url):
            os.symlink(url, filename)
        elif splittype(url)[0]:
            if download(url, filename) == None:
        if scheme in [ None, 'file' ]:
            if os.path.exists(path) == False:
                print "not found, skipping"
                continue
            elif name != path:
                if link_arg == False:
                    print "copying...",
                    shutil.copy2(path, name)
                else:
                    print "linking...",
                    os.symlink(path, name)
            else:
                pass
        elif scheme in [ 'http', 'https', 'ftp' ]:
            print "downloading...",
            name = download(url, file_arg)
            if name == None:
                print "failed"
                continue

        print "retrieved...",

        if validate(filename, hash):
            print "validated"
        print "validating...",
        if validate(name, hash_arg):
            print "ok"
            sys.exit(0)
        else:
            print "corrupt"

        try:
            os.remove(filename)
            os.remove(name)
        except OSError:
            pass