commit | author | age
|
de89cf
|
1 |
#!/usr/bin/python3.9 |
3a319e
|
2 |
# |
NJ |
3 |
# CDDL HEADER START |
|
4 |
# |
|
5 |
# The contents of this file are subject to the terms of the |
|
6 |
# Common Development and Distribution License (the "License"). |
|
7 |
# You may not use this file except in compliance with the License. |
|
8 |
# |
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
10 |
# or http://www.opensolaris.org/os/licensing. |
|
11 |
# See the License for the specific language governing permissions |
|
12 |
# and limitations under the License. |
|
13 |
# |
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
|
19 |
# |
|
20 |
# CDDL HEADER END |
|
21 |
# |
8beffa
|
22 |
# Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. |
3a319e
|
23 |
# |
NJ |
24 |
# |
fb10ba
|
25 |
# userland-fetch - a file download utility |
3a319e
|
26 |
# |
NJ |
27 |
# A simple program similiar to wget(1), but handles local file copy, ignores |
|
28 |
# directories, and verifies file hashes. |
|
29 |
# |
|
30 |
|
f8b2e5
|
31 |
import errno |
3a319e
|
32 |
import os |
NJ |
33 |
import sys |
6ddf48
|
34 |
import shutil |
8beffa
|
35 |
import json |
f8b2e5
|
36 |
import subprocess |
AŠ |
37 |
import re |
|
38 |
import gzip |
|
39 |
import bz2 |
222cf2
|
40 |
from urllib.parse import urlparse |
fb10ba
|
41 |
from urllib.request import urlopen |
222cf2
|
42 |
from urllib.error import HTTPError,URLError |
fb10ba
|
43 |
from urllib.request import Request |
222cf2
|
44 |
from pathlib import Path |
0e8406
|
45 |
import hashlib |
fb10ba
|
46 |
from http.client import BadStatusLine |
3a319e
|
47 |
|
222cf2
|
48 |
# EXIT CODES: |
D |
49 |
# 1 - unspecified error |
|
50 |
# 2 - download uses insecure protocol |
|
51 |
# 3 - was unable to find a suitable download |
|
52 |
# 4 - need-hash specified but no hash was found |
|
53 |
|
|
54 |
# NEW PARAMETERS: |
|
55 |
# -n/--need-hash: Set this to tell userland-fetch to fail if it cannot find a |
|
56 |
# correct hash. This also causes userland-fetch to search for |
|
57 |
# and download hash files, if they are not already present in |
|
58 |
# HASH_DIR. If --hash is provided this effectively does nothing. |
|
59 |
# |
|
60 |
# -N/--need-sig: Set this to tell userland-fetch to require a signature. This |
|
61 |
# also causes userland-fetch to search for signature files. If |
|
62 |
# the signature fails then the download is considered corrupted, |
|
63 |
# and will be deleted unless --keep is set. |
|
64 |
# This means that if the signature can't be checked, the file |
|
65 |
# WILL be deleted! |
|
66 |
# |
|
67 |
# -c/--clobber-hash: Set this to tell userland-fetch to clobber old hash files. |
|
68 |
# userland-fetch will replace hash files in HASH_DIR with their |
|
69 |
# remote counterparts. |
|
70 |
# |
|
71 |
|
|
72 |
# convert environment variables to global python variables |
|
73 |
def prep_envvars(): |
|
74 |
# This algorithm is set if it cannot be found in the filename |
|
75 |
global DEFAULT_HASH_ALGO |
|
76 |
DEFAULT_HASH_ALGO=os.getenv("DEFAULT_HASH_ALGO","sha256") |
|
77 |
|
|
78 |
global DEFAULT_HASH_FILES |
|
79 |
try: |
|
80 |
DEFAULT_HASH_FILES=[ x for x in os.environ["DEFAULT_HASH_FILES"].split(" ") if x ] |
|
81 |
except KeyError: |
|
82 |
DEFAULT_HASH_FILES=["SHA256SUMS","sha256sums.txt"] |
|
83 |
|
|
84 |
global HASH_DIR |
|
85 |
try: |
|
86 |
HASH_DIR = os.path.realpath(os.environ["HASH_DIR"]) |
|
87 |
except KeyError: |
|
88 |
# set after getting cmdline args |
|
89 |
HASH_DIR = None |
|
90 |
|
|
91 |
global SECURE_PROTOCOLS |
|
92 |
try: |
|
93 |
SECURE_PROTOCOLS=["UNCHECKED"]+[ x for x in os.environ["SECURE_PROTOCOLS"].split(" ") if x ] |
|
94 |
except KeyError: |
|
95 |
SECURE_PROTOCOLS=["UNCHECKED","https"] |
|
96 |
|
|
97 |
global SIGNATURE_EXTENSIONS |
|
98 |
try: |
|
99 |
SIGNATURE_EXTENSIONS=[ x for x in os.environ["SIGNATURE_EXTENSIONS"].split(" ") if x ] |
|
100 |
except KeyError: |
|
101 |
SIGNATURE_EXTENSIONS=["sig","asc"] |
|
102 |
|
|
103 |
global ALLOW_UNVERIFIED_DOWNLOADS |
|
104 |
try: |
|
105 |
ALLOW_UNVERIFIED_DOWNLOADS = os.environ["ALLOW_UNVERIFIED_DOWNLOADS"] == 'yes' |
|
106 |
except KeyError: |
|
107 |
ALLOW_UNVERIFIED_DOWNLOADS = False |
|
108 |
|
|
109 |
LOCAL_SCHEMES = [None, 'file',''] |
|
110 |
REMOTE_SCHEMES = ['https','http','ftp'] |
52916a
|
111 |
|
23ff40
|
112 |
def printIOError(e, txt): |
52916a
|
113 |
""" Function to decode and print IOError type exception """ |
fb10ba
|
114 |
print("I/O Error: " + txt + ": ") |
52916a
|
115 |
try: |
AŠ |
116 |
(code, message) = e |
fb10ba
|
117 |
print(str(message) + " (" + str(code) + ")") |
52916a
|
118 |
except: |
fb10ba
|
119 |
print(str(e)) |
52916a
|
120 |
|
222cf2
|
121 |
# TODO: refactor this so there aren't any global variables |
D |
122 |
VALIDATE_ERROR="" |
|
123 |
VALIDATE_CODE=-1 |
f8b2e5
|
124 |
def validate_signature(path, signature): |
AŠ |
125 |
"""Given paths to a file and a detached PGP signature, verify that |
|
126 |
the signature is valid for the file. Current configuration allows for |
|
127 |
unrecognized keys to be downloaded as necessary.""" |
|
128 |
|
|
129 |
# Find the root of the repo so that we can point GnuPG at the right |
|
130 |
# configuration and keyring. |
fb10ba
|
131 |
proc = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, |
AP |
132 |
universal_newlines=True) |
f8b2e5
|
133 |
proc.wait() |
AŠ |
134 |
if proc.returncode != 0: |
|
135 |
return False |
|
136 |
out, err = proc.communicate() |
|
137 |
gpgdir = os.path.join(out.strip(), "tools", ".gnupg") |
|
138 |
|
|
139 |
# Skip the permissions warning: none of the information here is private, |
|
140 |
# so not having to worry about getting git keeping the directory |
|
141 |
# unreadable is just simplest. |
|
142 |
try: |
|
143 |
proc = subprocess.Popen(["gpg2", "--verify", |
|
144 |
"--no-permission-warning", "--homedir", gpgdir, signature, |
|
145 |
path], stdin=open("/dev/null"), |
222cf2
|
146 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
fb10ba
|
147 |
universal_newlines=True) |
f8b2e5
|
148 |
except OSError as e: |
AŠ |
149 |
# If the executable simply couldn't be found, just skip the |
|
150 |
# validation. |
|
151 |
if e.errno == errno.ENOENT: |
|
152 |
return False |
|
153 |
raise |
|
154 |
|
|
155 |
proc.wait() |
222cf2
|
156 |
global VALIDATE_CODE |
D |
157 |
VALIDATE_CODE = proc.returncode |
f8b2e5
|
158 |
if proc.returncode != 0: |
AŠ |
159 |
# Only print GnuPG's output when there was a problem. |
222cf2
|
160 |
# Make this a global variable so we can output it somewhere tidy. |
D |
161 |
global VALIDATE_ERROR |
|
162 |
VALIDATE_ERROR=proc.stdout.read() |
f8b2e5
|
163 |
return False |
AŠ |
164 |
return True |
|
165 |
|
|
166 |
|
0e8406
|
167 |
def validate(file, hash): |
f8b2e5
|
168 |
"""Given a file-like object and a hash string, verify that the hash |
AŠ |
169 |
matches the file contents.""" |
|
170 |
|
|
171 |
try: |
|
172 |
algorithm, hashvalue = hash.split(':') |
|
173 |
except: |
222cf2
|
174 |
algorithm = DEFAULT_HASH_ALGO |
35e110
|
175 |
|
52916a
|
176 |
# force migration away from sha1 |
AŠ |
177 |
if algorithm == "sha1": |
222cf2
|
178 |
algorithm = DEFAULT_HASH_ALGO |
f8b2e5
|
179 |
|
52916a
|
180 |
try: |
AŠ |
181 |
m = hashlib.new(algorithm) |
|
182 |
except ValueError: |
222cf2
|
183 |
print("Unable to generate hashlib instance for",algorithm) |
52916a
|
184 |
return False |
3a319e
|
185 |
|
fb10ba
|
186 |
try: |
AP |
187 |
block = file.read() |
52916a
|
188 |
m.update(block) |
fb10ba
|
189 |
return "%s:%s" % (algorithm, m.hexdigest()) |
AP |
190 |
except IOError as err: |
|
191 |
print(str(err), end=' ') |
222cf2
|
192 |
except EOFError as err: |
D |
193 |
print(str(err), end=' ') |
3a319e
|
194 |
|
fb10ba
|
195 |
return "%s:" % (algorithm) |
52916a
|
196 |
|
0e8406
|
197 |
|
NJ |
198 |
def validate_container(filename, hash): |
f8b2e5
|
199 |
"""Given a file path and a hash string, verify that the hash matches the |
AŠ |
200 |
file contents.""" |
|
201 |
|
52916a
|
202 |
try: |
fb10ba
|
203 |
file = open(filename, 'rb') |
52916a
|
204 |
except IOError as e: |
AŠ |
205 |
printIOError(e, "Can't open file " + filename) |
|
206 |
return False |
|
207 |
return validate(file, hash) |
0e8406
|
208 |
|
NJ |
209 |
|
|
210 |
def validate_payload(filename, hash): |
f8b2e5
|
211 |
"""Given a file path and a hash string, verify that the hash matches the |
AŠ |
212 |
payload (uncompressed content) of the file.""" |
0e8406
|
213 |
|
222cf2
|
214 |
expr_bz = re.compile('.+\.bz2?$', re.IGNORECASE) |
52916a
|
215 |
expr_gz = re.compile('.+\.gz$', re.IGNORECASE) |
AŠ |
216 |
expr_tgz = re.compile('.+\.tgz$', re.IGNORECASE) |
222cf2
|
217 |
expr_tbz = re.compile('.+\.tbz2?$', re.IGNORECASE) |
0e8406
|
218 |
|
52916a
|
219 |
try: |
AŠ |
220 |
if expr_bz.match(filename): |
fb10ba
|
221 |
file = bz2.BZ2File(filename, 'rb') |
52916a
|
222 |
elif expr_gz.match(filename): |
fb10ba
|
223 |
file = gzip.GzipFile(filename, 'rb') |
52916a
|
224 |
elif expr_tgz.match(filename): |
fb10ba
|
225 |
file = gzip.GzipFile(filename, 'rb') |
222cf2
|
226 |
elif expr_tbz.match(filename): |
D |
227 |
file = bz2.GzipFile(filename, 'rb') |
52916a
|
228 |
else: |
AŠ |
229 |
return False |
|
230 |
except IOError as e: |
|
231 |
printIOError(e, "Can't open archive " + filename) |
|
232 |
return False |
|
233 |
return validate(file, hash) |
0e8406
|
234 |
|
3a319e
|
235 |
|
222cf2
|
236 |
def download(url, filename=None, user_agent_arg=None, quiet=None,allow_partial=True): |
f8b2e5
|
237 |
"""Download the content at the given URL to the given filename |
AŠ |
238 |
(defaulting to the basename of the URL if not given. If 'quiet' is |
|
239 |
True, throw away any error messages. Returns the name of the file to |
|
240 |
which the content was donloaded.""" |
222cf2
|
241 |
retval = None |
52916a
|
242 |
try: |
222cf2
|
243 |
req = Request(url,method="HEAD") |
f8b2e5
|
244 |
if user_agent_arg is not None: |
52916a
|
245 |
req.add_header("User-Agent", user_agent_arg) |
222cf2
|
246 |
if filename is None: |
D |
247 |
filename = req.get_full_url().split('/')[-1] |
|
248 |
retry = 1 |
|
249 |
dl = 0 |
|
250 |
i = urlopen(req) |
|
251 |
if 'transfer-encoding' in i.headers and i.headers['transfer-encoding'] == 'chunked': |
|
252 |
length = 0 |
|
253 |
if not quiet: |
|
254 |
print("length unknown (streamed/chunked)") |
|
255 |
else: |
|
256 |
try: |
|
257 |
length = int(i.headers['content-length']) |
|
258 |
if not quiet: |
|
259 |
print("length %i bytes" % (length)) |
|
260 |
except (KeyError,ValueError,TypeError): |
|
261 |
length = 0 |
|
262 |
if not quiet: |
|
263 |
print("length unknown") |
|
264 |
if not 'accept-ranges' in i.headers or i.headers['accept-ranges'] != 'bytes': |
|
265 |
if not quiet: |
|
266 |
print("No partial download support from server") |
|
267 |
allow_partial = False |
|
268 |
i.close() |
|
269 |
req.method = "GET" |
|
270 |
# This might speed things up and keep memory usage down |
|
271 |
while retry <= 3: |
|
272 |
with open(filename + ".part","ab" if allow_partial else "wb") as o: |
|
273 |
try: |
|
274 |
# seek to end of the file if applicable |
|
275 |
if allow_partial: |
|
276 |
o.seek(0,2) |
|
277 |
dl = o.tell() |
|
278 |
if not quiet: |
|
279 |
print("(Attempt %i of 3%s)..." % (retry,"; %i bytes done"%(dl) if dl else ""),end=" ") |
|
280 |
if dl > 0: |
|
281 |
req.add_header("Range","bytes=%i-"%(dl)) |
|
282 |
with urlopen(req) as i: |
|
283 |
src = i.read(65536) |
|
284 |
while len(src) > 0: |
|
285 |
o.write(src) |
|
286 |
src = i.read(65536) |
|
287 |
retry = 4 |
|
288 |
if length > 0 and o.tell() < length: |
|
289 |
if not quiet: |
|
290 |
print("Download of %s stopped abruptly." % (str(url))) |
|
291 |
retry += 1 |
|
292 |
|
|
293 |
except URLError as e: |
|
294 |
if not quiet: |
|
295 |
print("Error downloading %s at %i bytes: %s" % (str(url),dl,str(e))) |
|
296 |
# if we haven't downloaded any bytes since the last URLError, cancel the download. |
|
297 |
if dl > 0 and o.tell() > dl: |
|
298 |
dl = o.tell() |
|
299 |
retry += 1 |
|
300 |
req.add_header("Range","bytes=%i-"%(o.tell()+1)) |
|
301 |
else: |
|
302 |
retry = 4 |
|
303 |
except HTTPError as e: |
|
304 |
if not quiet: |
|
305 |
print("Error downloading %s: %s" % (str(url),str(e))) |
|
306 |
retry = 4 |
|
307 |
# return the name of the file that we downloaded the data to. |
|
308 |
os.rename(filename+".part",filename) |
|
309 |
retval = filename |
52916a
|
310 |
except IOError as e: |
f8b2e5
|
311 |
if not quiet: |
AŠ |
312 |
printIOError(e, "Can't open url " + url) |
1e1f59
|
313 |
except BadStatusLine as e: |
AP |
314 |
if not quiet: |
|
315 |
print("Can't open url %s: server answered with code which we couldn't understand " % (url)) |
222cf2
|
316 |
except KeyboardInterrupt: |
D |
317 |
print("Cancelling download...") |
3a319e
|
318 |
|
222cf2
|
319 |
return retval |
52916a
|
320 |
|
3a319e
|
321 |
|
222cf2
|
322 |
def download_paths(search, filenames, url): |
f8b2e5
|
323 |
"""Returns a list of URLs where the file 'filename' might be found, |
AŠ |
324 |
using 'url', 'search', and $DOWNLOAD_SEARCH_PATH as places to look. |
|
325 |
|
|
326 |
If 'filename' is None, then the list will simply contain 'url'.""" |
|
327 |
|
52916a
|
328 |
urls = list() |
222cf2
|
329 |
if type(filenames) == str: |
D |
330 |
filenames = [filenames] |
3a319e
|
331 |
|
222cf2
|
332 |
if filenames is not None: |
52916a
|
333 |
tmp = os.getenv('DOWNLOAD_SEARCH_PATH') |
AŠ |
334 |
if tmp: |
|
335 |
search += tmp.split(' ') |
222cf2
|
336 |
for filename in filenames: |
D |
337 |
file = os.path.basename(filename) |
6ddf48
|
338 |
|
222cf2
|
339 |
urls += [base + '/' + file for base in search] |
6ddf48
|
340 |
|
222cf2
|
341 |
# filename should always be first |
D |
342 |
if filename in urls: |
|
343 |
urls.remove(filename) |
|
344 |
urls.insert(0, filename) |
6ddf48
|
345 |
|
52916a
|
346 |
# command line url is a fallback, so it's last |
f8b2e5
|
347 |
if url is not None and url not in urls: |
222cf2
|
348 |
parse_result = urlparse(url) |
D |
349 |
scheme = parse_result.scheme |
|
350 |
path = parse_result.path |
8beffa
|
351 |
if scheme == "pypi": |
DD |
352 |
url = pypi_url(url, os.path.basename(filename)) |
|
353 |
if url != None and url not in urls: |
|
354 |
urls.append(url) |
3a319e
|
355 |
|
58825e
|
356 |
# last resort path |
222cf2
|
357 |
if filenames is not None: |
58825e
|
358 |
tmp = os.getenv('DOWNLOAD_FALLBACK_PATH') |
AP |
359 |
if tmp: |
222cf2
|
360 |
for filename in filenames: |
D |
361 |
file = os.path.basename(filename) |
|
362 |
urls += [base + '/' + file for base in tmp.split(' ')] |
58825e
|
363 |
|
222cf2
|
364 |
local_urls = list() |
D |
365 |
remote_urls = list() |
|
366 |
# sort entries by local first, then remote: |
|
367 |
for entry in urls: |
|
368 |
if urlparse(entry).scheme in LOCAL_SCHEMES: |
|
369 |
local_urls.append(entry) |
|
370 |
else: |
|
371 |
remote_urls.append(entry) |
|
372 |
return local_urls + remote_urls |
58825e
|
373 |
|
52916a
|
374 |
|
8beffa
|
375 |
def pypi_url(url, filename): |
DD |
376 |
"""Given a pypi: URL, return the real URL for that component/version. |
|
377 |
|
|
378 |
The pypi scheme has a host (with an empty host defaulting to |
|
379 |
pypi.python.org), and a path that should be of the form |
|
380 |
"component==version". Other specs could be supported, but == is the |
|
381 |
only thing that makes sense in this context. |
|
382 |
|
|
383 |
The filename argument is the name of the expected file to download, so |
|
384 |
that when pypi gives us multiple archives to choose from, we can pick |
|
385 |
the right one. |
|
386 |
""" |
|
387 |
|
222cf2
|
388 |
|
D |
389 |
parse_result = urlparse(url) |
|
390 |
host = parse_result.netloc |
|
391 |
path = parse_result.path |
8beffa
|
392 |
|
DD |
393 |
# We have to use ==; anything fancier would require pkg_resources, but |
|
394 |
# really that's the only thing that makes sense in this context. |
|
395 |
try: |
222cf2
|
396 |
name, version = re.match("/(.*)==(.*)$", path).groups() |
8beffa
|
397 |
except AttributeError: |
fb10ba
|
398 |
print("PyPI URLs must be of the form 'pypi:///component==version'") |
8beffa
|
399 |
return None |
DD |
400 |
|
|
401 |
if not host: |
|
402 |
jsurl = "https://pypi.python.org/pypi/%s/json" % name |
|
403 |
else: |
|
404 |
jsurl = "https://%s/pypi/%s/json" % (host, name) |
|
405 |
|
|
406 |
try: |
fb10ba
|
407 |
f = urlopen(jsurl, data=None) |
8beffa
|
408 |
except HTTPError as e: |
DD |
409 |
if e.getcode() == 404: |
fb10ba
|
410 |
print("Unknown component '%s'" % name) |
8beffa
|
411 |
else: |
DD |
412 |
printIOError(e, "Can't open PyPI JSON url %s" % url) |
|
413 |
return None |
|
414 |
except IOError as e: |
|
415 |
printIOError(e, "Can't open PyPI JSON url %s" % url) |
|
416 |
return None |
fb10ba
|
417 |
content = f.read().decode("utf-8") |
AP |
418 |
js = json.loads(content) |
8beffa
|
419 |
try: |
DD |
420 |
verblock = js["releases"][version] |
|
421 |
except KeyError: |
fb10ba
|
422 |
print("Unknown version '%s'" % version) |
8beffa
|
423 |
return None |
DD |
424 |
|
|
425 |
urls = [ d["url"] for d in verblock ] |
|
426 |
for archiveurl in urls: |
|
427 |
if archiveurl.endswith("/%s" % filename): |
|
428 |
return archiveurl |
|
429 |
|
|
430 |
if urls: |
fb10ba
|
431 |
print("None of the following URLs delivers '%s':" % filename) |
AP |
432 |
print(" " + "\n ".join(urls)) |
8beffa
|
433 |
else: |
fb10ba
|
434 |
print("Couldn't find any suitable URLs") |
8beffa
|
435 |
return None |
3a319e
|
436 |
|
222cf2
|
437 |
def download_from_paths(search_list, file_arg, url, link_arg, quiet=False,get_signature=False,download_dir=None): |
f8b2e5
|
438 |
"""Attempts to download a file from a number of possible locations. |
AŠ |
439 |
Generates a list of paths where the file ends up on the local |
|
440 |
filesystem. This is a generator because while a download might be |
|
441 |
successful, the signature or hash may not validate, and the caller may |
|
442 |
want to try again from the next location. The 'link_arg' argument is a |
|
443 |
boolean which, when True, specifies that if the source is not a remote |
|
444 |
URL and not already found where it should be, to make a symlink to the |
|
445 |
source rather than copying it.""" |
|
446 |
|
|
447 |
for url in download_paths(search_list, file_arg, url): |
|
448 |
if not quiet: |
fb10ba
|
449 |
print("Source %s..." % url, end=' ') |
222cf2
|
450 |
elif quiet == 2: |
D |
451 |
if len(url) > 53: |
|
452 |
p = url[:24] + ' ... ' + url[-24:] |
|
453 |
else: |
|
454 |
p = url |
|
455 |
print(" {:54s}".format(p), end='') |
f8b2e5
|
456 |
|
222cf2
|
457 |
parse_result = urlparse(url) |
D |
458 |
scheme = parse_result.scheme |
|
459 |
path = parse_result.path |
|
460 |
|
f8b2e5
|
461 |
|
222cf2
|
462 |
if scheme in LOCAL_SCHEMES: |
D |
463 |
name = None |
|
464 |
if type(file_arg) == str: |
|
465 |
names = [file_arg] |
|
466 |
else: |
|
467 |
names = file_arg |
|
468 |
notfound = False |
|
469 |
for n in names: |
|
470 |
# don't rename stuff - there shouldn't be a file list here anyway |
|
471 |
if os.path.basename(n) != os.path.basename(url): |
|
472 |
continue |
|
473 |
if os.path.exists(path) is False: |
|
474 |
notfound = True |
f8b2e5
|
475 |
if not quiet: |
222cf2
|
476 |
print("not found, skipping file copy") |
D |
477 |
elif quiet == 2: |
|
478 |
print("{:10s}".format("-")) |
|
479 |
break |
|
480 |
elif n and n != path: |
|
481 |
if link_arg is False: |
|
482 |
if not quiet: |
|
483 |
print("\n copying...") |
|
484 |
shutil.copy2(path, n) |
|
485 |
else: |
|
486 |
if not quiet: |
|
487 |
print("\n linking...") |
|
488 |
os.symlink(path, n) |
f8b2e5
|
489 |
else: |
222cf2
|
490 |
name = n |
f8b2e5
|
491 |
if not quiet: |
222cf2
|
492 |
print("cached") |
D |
493 |
elif quiet == 2: |
|
494 |
print("{:10s}".format("cached"),end="") |
|
495 |
break |
|
496 |
if notfound: |
|
497 |
continue |
|
498 |
elif scheme in REMOTE_SCHEMES: |
f8b2e5
|
499 |
if not quiet: |
fb10ba
|
500 |
print("\n downloading...", end=' ') |
222cf2
|
501 |
if type(file_arg) == str: |
D |
502 |
name = download(url, file_arg, quiet,(scheme != 'ftp')) |
|
503 |
else: |
|
504 |
if not download_dir: |
|
505 |
download_dir = os.curdir |
|
506 |
name = download(url, os.path.join(download_dir,os.path.basename(url)),quiet,(scheme != 'ftp')) |
|
507 |
if get_signature and name: |
|
508 |
for ext in SIGNATURE_EXTENSIONS: |
|
509 |
sig = download(url+"."+ext, name+"."+ext, quiet,(scheme != 'ftp')) |
|
510 |
if sig: |
|
511 |
break |
f8b2e5
|
512 |
if name is None: |
AŠ |
513 |
if not quiet: |
fb10ba
|
514 |
print("failed") |
222cf2
|
515 |
elif quiet == 2: |
D |
516 |
print("{:10s}".format("-")) |
f8b2e5
|
517 |
continue |
222cf2
|
518 |
else: |
D |
519 |
if not quiet: |
|
520 |
print("ok") |
|
521 |
elif quiet == 2: |
|
522 |
print("{:10s}".format("fetched"),end="") |
|
523 |
yield (name,url) |
f8b2e5
|
524 |
|
AŠ |
525 |
|
222cf2
|
526 |
def find_hash_in_file(filename,hash_file): |
D |
527 |
splits = hash_file.split('.') |
7c4acf
|
528 |
regex = re.compile('([0-9a-fA-F]+)( [ \*](.*/)?)('+os.path.basename(filename)+'$)') |
D |
529 |
match = re.match("(^[a-z0-9]+)(sums?(.txt)?$)",hash_file.lower()) |
222cf2
|
530 |
if '.'.join(splits[:-1]) == filename: |
7c4acf
|
531 |
algo = re.match('([a-zA-Z0-9]+)(sums?)',hash_file.split('.')[-1]).group(1) |
222cf2
|
532 |
elif match: |
D |
533 |
algo = match.group(1) |
|
534 |
else: |
|
535 |
algo = DEFAULT_HASH_ALGO |
|
536 |
with open(os.path.join(HASH_DIR,hash_file),"r") as file: |
|
537 |
hash_value = None |
|
538 |
for line in file.readlines(): |
|
539 |
hash_value = regex.match(line) |
|
540 |
if hash_value is not None: |
|
541 |
hash_value = hash_value.group(1) |
|
542 |
break |
|
543 |
if hash_value is not None: |
|
544 |
return "%s:%s" % (algo,hash_value) |
|
545 |
return None |
|
546 |
|
|
547 |
def find_hash_in_hash_dir(filename): |
|
548 |
try: |
|
549 |
hash_value = None |
|
550 |
if not os.path.exists(HASH_DIR): |
|
551 |
return None, None |
|
552 |
for hash_file in sorted(os.listdir(HASH_DIR)): |
|
553 |
splits = hash_file.split('.') |
|
554 |
if '.'.join(splits[:-1]) in SIGNATURE_EXTENSIONS: |
|
555 |
continue |
|
556 |
hash_value = find_hash_in_file(filename,hash_file) |
|
557 |
if hash_value: |
|
558 |
return hash_value, hash_file |
|
559 |
return None, None |
|
560 |
except NotADirectoryError: |
|
561 |
print(HASH_DIR,"should be a directory containing hashfiles in the",DEFAULT_HASH_ALGO+"sum","format.") |
|
562 |
return (1) |
|
563 |
except IsADirectoryError: |
|
564 |
print(hash_file,"should be a file containing hashes, not a directory.") |
|
565 |
return 1 |
3a319e
|
566 |
def usage(): |
fb10ba
|
567 |
print("Usage: %s [-a|--user-agent (user-agent)] [-f|--file (file)] [-l|--link] " \ |
222cf2
|
568 |
"[-k|--keep] [-h|--hash (hash)] [-n|--need-hash] [-s|--search (search-dir)] " \ |
D |
569 |
"[-g|--get-hashes] [-G|--get-sigs] " \ |
291e6d
|
570 |
"[-S|--sigurl (signature-url)] [-N|--need-sig] --url (url)" % (sys.argv[0].split('/')[-1])) |
222cf2
|
571 |
return 1 |
52916a
|
572 |
|
3a319e
|
573 |
|
NJ |
574 |
def main(): |
52916a
|
575 |
import getopt |
fb10ba
|
576 |
sys.stdout.flush() |
222cf2
|
577 |
try: |
D |
578 |
opts, args = getopt.getopt(sys.argv[1:], "a:f:h:lks:u:GgNnc", |
|
579 |
["file=", "link", "keep", "hash=", "search=", "url=","get-sigs","get-hashes", |
|
580 |
"sigurl=", "user-agent=", "need-sig", "need-hash","clobber-hash"]) |
|
581 |
sys.exit(realmain(opts, args)) |
|
582 |
except getopt.GetoptError as err: |
|
583 |
print(str(err)) |
|
584 |
usage() |
3a319e
|
585 |
|
222cf2
|
586 |
def realmain(opts, args): |
D |
587 |
prep_envvars() |
52916a
|
588 |
user_agent_arg = None |
AŠ |
589 |
file_arg = None |
|
590 |
link_arg = False |
|
591 |
keep_arg = False |
|
592 |
hash_arg = None |
|
593 |
url_arg = None |
f8b2e5
|
594 |
sig_arg = None |
291e6d
|
595 |
need_sig = False |
222cf2
|
596 |
get_signature = False |
D |
597 |
need_hash = False |
|
598 |
get_hash = False |
|
599 |
clobber_hash = False |
52916a
|
600 |
search_list = list() |
222cf2
|
601 |
retval = 1 |
3a319e
|
602 |
|
52916a
|
603 |
for opt, arg in opts: |
AŠ |
604 |
if opt in ["-a", "--user-agent"]: |
|
605 |
user_agent_arg = arg |
|
606 |
elif opt in ["-f", "--file"]: |
|
607 |
file_arg = arg |
|
608 |
elif opt in ["-l", "--link"]: |
|
609 |
link_arg = True |
|
610 |
elif opt in ["-k", "--keep"]: |
|
611 |
keep_arg = True |
|
612 |
elif opt in ["-h", "--hash"]: |
|
613 |
hash_arg = arg |
222cf2
|
614 |
elif opt in ["-n", "--need-hash"]: |
D |
615 |
need_hash = True |
|
616 |
elif opt in ["-g", "--get-hashes"]: |
|
617 |
get_hash = True |
52916a
|
618 |
elif opt in ["-s", "--search"]: |
AŠ |
619 |
search_list.append(arg) |
f8b2e5
|
620 |
elif opt in ["-S", "--sigurl"]: |
AŠ |
621 |
sig_arg = arg |
52916a
|
622 |
elif opt in ["-u", "--url"]: |
AŠ |
623 |
url_arg = arg |
291e6d
|
624 |
elif opt in ["-N", "--need-sig"]: |
D |
625 |
need_sig = True |
222cf2
|
626 |
elif opt in ["-G", "--get-sigs"]: |
D |
627 |
get_signature = True |
|
628 |
elif opt in ["-c", "--clobber-hash"]: |
|
629 |
clobber_hash = True |
52916a
|
630 |
else: |
AŠ |
631 |
assert False, "unknown option" |
3a319e
|
632 |
|
f8b2e5
|
633 |
if url_arg is None: |
222cf2
|
634 |
if clobber_hash and len(search_list) == 0: |
D |
635 |
print("WARN: -c/--clobber-hash is meaningless without --search or --url. Ignoring.") |
|
636 |
clobber_hash = False |
|
637 |
if file_arg is None: |
|
638 |
usage() |
|
639 |
scheme = "UNCHECKED" |
|
640 |
else: |
|
641 |
parse_result = urlparse(url_arg) |
|
642 |
scheme = parse_result.scheme |
|
643 |
path = parse_result.path |
|
644 |
if file_arg is None: |
|
645 |
file_arg = os.path.realpath(os.path.join(os.curdir,os.path.basename(path))) |
3a319e
|
646 |
|
222cf2
|
647 |
file_arg = os.path.realpath(file_arg) |
D |
648 |
filename = os.path.basename(file_arg) |
|
649 |
global HASH_DIR |
|
650 |
if not HASH_DIR: |
|
651 |
HASH_DIR = os.path.realpath(os.path.join(os.path.dirname(file_arg),"hashes")) |
|
652 |
valid_sig = False |
3a319e
|
653 |
|
222cf2
|
654 |
if clobber_hash or get_hash: |
D |
655 |
print("Hash directory: %s [clobbering: %s]" % (HASH_DIR,str(clobber_hash))) |
|
656 |
if clobber_hash: |
|
657 |
HASH_DIR_ORIG = HASH_DIR |
|
658 |
HASH_DIR = HASH_DIR + ".tmp" |
|
659 |
try: |
|
660 |
os.mkdir(HASH_DIR) |
|
661 |
except FileNotFoundError: |
|
662 |
print("Refusing to create %s recursively - is HASH_DIR set correctly?" % (HASH_DIR)) |
|
663 |
return 1 |
|
664 |
except FileExistsError: |
|
665 |
pass |
|
666 |
|
|
667 |
# We need to account for the following possibilities for hash files: |
|
668 |
# 1: .asc with embedded checksums (1 file, needs PGP stripping) |
|
669 |
# TODO: ^ |
|
670 |
# 2: .asc or .sig, detached from hash file (2 files) |
|
671 |
# 3: checksums without signature (need a secure protocol) |
|
672 |
|
|
673 |
print("Sourcing hash files... ",end="") |
|
674 |
search_hash_files = DEFAULT_HASH_FILES + [ |
|
675 |
filename + '.' + DEFAULT_HASH_ALGO, |
|
676 |
filename + '.' + DEFAULT_HASH_ALGO + 'sum', |
|
677 |
filename + '.' + DEFAULT_HASH_ALGO + 'sums' |
|
678 |
] |
|
679 |
hash_file = None |
|
680 |
print("\n {:54s}{:10s}{:10s}".format("URL","LOCALITY","HAS HASH")) |
|
681 |
if url_arg: |
|
682 |
if not search_list: |
|
683 |
search_list = [] |
|
684 |
search_list.append(os.path.dirname(url_arg)) |
|
685 |
search_hash_files = [ os.path.join(HASH_DIR,x) for x in search_hash_files ] |
|
686 |
for hashname, hashurl in download_from_paths(search_list, search_hash_files , None, link_arg, quiet=2,get_signature=True,download_dir=HASH_DIR): |
|
687 |
scheme = urlparse(hashurl).scheme |
|
688 |
safe = scheme in SECURE_PROTOCOLS or scheme in LOCAL_SCHEMES |
|
689 |
valid_sig = False |
|
690 |
for sigext in SIGNATURE_EXTENSIONS: |
|
691 |
signame = hashname + "." + sigext |
|
692 |
if os.path.exists(signame): |
|
693 |
valid_sig = validate_signature(hashname,signame) |
|
694 |
if not valid_sig and (not safe or need_sig): |
|
695 |
print("denied (hashfile download did not meet security criteria)") |
|
696 |
os.remove(hashname) |
|
697 |
Path(signame).unlink(missing_ok=True) |
|
698 |
continue |
|
699 |
hash_arg = find_hash_in_file(filename, hashname) |
|
700 |
if not hash_arg: |
|
701 |
print("no") |
52916a
|
702 |
else: |
222cf2
|
703 |
print("YES") |
D |
704 |
hash_file = hashname |
|
705 |
break |
|
706 |
if hash_file is not None: |
|
707 |
print("INFO: hash found for",filename,"in",hash_file) |
3a319e
|
708 |
|
222cf2
|
709 |
if clobber_hash: |
D |
710 |
for file in os.listdir(HASH_DIR): |
|
711 |
orig_file = None |
|
712 |
new_file = None |
|
713 |
try: |
|
714 |
orig_file = os.path.join(HASH_DIR_ORIG,os.path.basename(file)) |
|
715 |
new_file = os.path.join(HASH_DIR,file) |
|
716 |
os.rename(new_file,orig_file) |
|
717 |
except IsADirectoryError as e: |
|
718 |
print("ERROR: moving hashfiles to HASH_DIR failed: %s" % (str(e))) |
|
719 |
except OSError as e: |
|
720 |
print("OSError: %s (%s -> %s)" %(str(e),new_file,orig_file)) |
|
721 |
try: |
|
722 |
os.rmdir(HASH_DIR) |
|
723 |
except OSError as e: |
|
724 |
print("Couldn't remove %s: %s" % (HASH_DIR,str(e))) |
|
725 |
HASH_DIR = HASH_DIR_ORIG |
|
726 |
elif hash_arg == None: |
|
727 |
hash_arg, hash_file = find_hash_in_hash_dir(filename) |
|
728 |
if hash_file is not None: |
|
729 |
print("INFO: hash found for",filename,"in",hash_file) |
|
730 |
else: |
|
731 |
print("INFO: not using any hashes in %s for" % (HASH_DIR),filename,"(overridden with --hash)") |
f8b2e5
|
732 |
|
222cf2
|
733 |
|
D |
734 |
|
|
735 |
if (hash_arg is None or hash_arg == 'none') and need_hash: |
|
736 |
print("-n/--need-hash and no hash found. Exiting.") |
|
737 |
return 4 |
|
738 |
if ALLOW_UNVERIFIED_DOWNLOADS: |
|
739 |
print("WARN: ALLOW_UNVERIFIED_DOWNLOADS set.") |
|
740 |
|
|
741 |
if sig_arg: |
|
742 |
if get_signature: |
|
743 |
print("INFO: not searching with -g (--sigurl provided)") |
|
744 |
get_signature=False |
|
745 |
for name, url in download_from_paths(search_list, file_arg, url_arg, link_arg,get_signature=get_signature): |
|
746 |
scheme = urlparse(url).scheme |
|
747 |
if not name: |
|
748 |
print(" was not downloaded") |
|
749 |
continue |
|
750 |
print(" validating signature...", end=' ') |
|
751 |
if valid_sig: |
|
752 |
print("hashfile had valid signature") |
52916a
|
753 |
else: |
222cf2
|
754 |
sig_file = None |
D |
755 |
if sig_arg: |
|
756 |
if sig_arg == 'none': |
|
757 |
print("skipping (--sigurl none)") |
|
758 |
else: |
|
759 |
print("using %s..." % sig_arg,end=' ') |
|
760 |
if urlparse(sig_arg).scheme in REMOTE_SCHEMES: |
|
761 |
sig_file = download(sig_arg,filename=os.path.join(os.path.dirname(name),os.path.basename(sig_arg)),quiet=True,allow_partial=False) |
|
762 |
else: |
|
763 |
if get_signature: |
|
764 |
print("checking remote signature...",end=' ') |
|
765 |
else: |
|
766 |
print("checking local signature...",end=' ') |
|
767 |
errors = [] |
|
768 |
if not sig_file: |
|
769 |
for ext in SIGNATURE_EXTENSIONS: |
|
770 |
if os.path.exists(name+'.'+ext): |
|
771 |
sig_file = name+'.'+ext |
|
772 |
break |
|
773 |
if sig_file: |
|
774 |
if validate_signature(name, sig_file): |
|
775 |
print("VALID") |
|
776 |
valid_sig = True |
|
777 |
else: |
|
778 |
print("invalid") |
|
779 |
errors.append((sig_file,VALIDATE_CODE,VALIDATE_ERROR)) |
|
780 |
break |
|
781 |
else: |
|
782 |
print("not found") |
|
783 |
|
|
784 |
if not valid_sig: |
|
785 |
print(" signature validation failed\n") |
|
786 |
bad_signature = False |
|
787 |
for error in errors: |
|
788 |
print("---%s output(exit code %d):\n%s---" % error) |
|
789 |
if error[1] == 1: |
|
790 |
bad_signature = True |
|
791 |
if need_sig: |
|
792 |
if not keep_arg or bad_signature: |
|
793 |
print("WARN: Deleting corrupt file.") |
|
794 |
try: |
|
795 |
os.remove(name) |
|
796 |
except OSError: |
|
797 |
pass |
|
798 |
print("-N/--need-sig is set. This download cannot be used.") |
|
799 |
retval = 3 |
|
800 |
continue |
|
801 |
elif not need_hash: |
|
802 |
retval = 0 |
|
803 |
break |
|
804 |
print(" validating hash...", end=' ') |
|
805 |
if hash_arg and hash_arg != 'none': |
|
806 |
realhash = validate_container(name, hash_arg) |
|
807 |
else: |
|
808 |
realhash = "skipped calculation (--hash none)" |
|
809 |
hash_arg = None |
|
810 |
|
|
811 |
if realhash == hash_arg: |
|
812 |
print("ok") |
|
813 |
retval = 0 |
|
814 |
else: |
|
815 |
if hash_arg and hash_arg != 'none': |
|
816 |
payloadhash = validate_payload(name, hash_arg) |
|
817 |
else: |
|
818 |
payloadhash = "skipped calculation (--hash none)" |
52916a
|
819 |
if payloadhash == hash_arg: |
fb10ba
|
820 |
print("ok") |
222cf2
|
821 |
retval = 0 |
f8b2e5
|
822 |
else: |
222cf2
|
823 |
if not hash_arg or hash_arg == 'none': |
D |
824 |
scheme = urlparse(url).scheme |
|
825 |
if not ALLOW_UNVERIFIED_DOWNLOADS: |
|
826 |
print("ERROR: Cannot validate download (no hash or signature).") |
|
827 |
if keep_arg == False: |
|
828 |
try: |
|
829 |
print("\nWARN: Removing the downloaded file") |
|
830 |
os.remove(name) |
|
831 |
except OSError: |
|
832 |
pass |
|
833 |
retval = 3 |
|
834 |
continue |
|
835 |
elif scheme not in SECURE_PROTOCOLS and scheme not in LOCAL_SCHEMES: |
|
836 |
print("ERROR: This download uses an insecure protocol: '%s'." % (str(scheme),)) |
|
837 |
if keep_arg == False: |
|
838 |
try: |
|
839 |
print("\nWARN: Removing the downloaded file") |
|
840 |
os.remove(name) |
|
841 |
except OSError: |
|
842 |
pass |
|
843 |
retval = 2 |
|
844 |
continue |
|
845 |
print("ignoring errors") |
|
846 |
retval = 0 |
f8b2e5
|
847 |
else: |
222cf2
|
848 |
print("invalid hash!") |
D |
849 |
print(" expected: %s" % hash_arg) |
|
850 |
print(" actual: %s" % realhash) |
|
851 |
print(" payload: %s" % payloadhash) |
3a319e
|
852 |
|
222cf2
|
853 |
if valid_sig: |
D |
854 |
if need_hash: |
|
855 |
print("-n/--need-hash is set. This download cannot be used.") |
|
856 |
if keep_arg == False: |
|
857 |
try: |
|
858 |
print("\nWARN: Removing the downloaded file") |
|
859 |
os.remove(name) |
|
860 |
except OSError: |
|
861 |
pass |
|
862 |
retval = 3 |
|
863 |
continue |
52916a
|
864 |
|
222cf2
|
865 |
# If the signature validated, then we assume |
D |
866 |
# that the expected hash is just a typo. |
eefa3b
|
867 |
|
222cf2
|
868 |
# An invalid hash shouldn't cause us to remove |
D |
869 |
# the target file if the signature was valid. |
|
870 |
# Also, if the developer is in progress of upgrading |
|
871 |
# some package version or introduces a new one, and |
|
872 |
# explicitly ran "gmake fetch", keep the downloaded |
|
873 |
# file (Makefile is not in position to have a valid |
|
874 |
# checksum entry just yet) so it does not have to be |
|
875 |
# downloaded twice. |
|
876 |
retval = 0 |
eefa3b
|
877 |
else: |
222cf2
|
878 |
print("ERROR: This download failed to validate.") |
D |
879 |
if keep_arg == False: |
|
880 |
try: |
|
881 |
print("\nWARN: Removing the corrupt downloaded file") |
|
882 |
os.remove(name) |
|
883 |
except OSError: |
|
884 |
pass |
|
885 |
retval = 3 |
|
886 |
continue |
|
887 |
if retval == 0: |
|
888 |
break |
|
889 |
return retval |
f8b2e5
|
890 |
|
3a319e
|
891 |
if __name__ == "__main__": |
52916a
|
892 |
main() |