#!/usr/local/autopkg/python # # Copyright 2021 James Stewart @JGStew # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """See docstring for URLDownloaderPython class""" import json import os import ssl from hashlib import md5, sha1, sha256 from urllib.request import Request, urlopen import certifi from autopkglib.URLDownloader import URLDownloader __all__ = ["URLDownloaderPython"] class URLDownloaderPython(URLDownloader): """This is meant to be a pure python replacement for URLDownloader Prefetch Filename is still using URLDownloader's curl methods, rather than python. See: https://github.com/autopkg/autopkg/blob/master/Code/autopkglib/URLDownloader.py """ description = __doc__ input_variables = { "url": {"required": True, "description": "The URL to download."}, "download_dir": { "required": False, "description": ( "The directory where the file will be downloaded to. Defaults " "to RECIPE_CACHE_DIR/downloads." ), }, "filename": { "required": False, "description": "Filename to override the URL's tail.", }, "prefetch_filename": { "default": False, "required": False, "description": ( "If True, URLDownloader attempts to determine filename from HTTP " "headers downloaded before the file itself. 'prefetch_filename' " "overrides 'filename' option. Filename is determined from the first " "available source of information in this order:\n" "\t1. Content-Disposition header\n" "\t2. Location header\n" "\t3. 'filename' option (if set)\n" "\t4. last part of 'url'. \n" "'prefetch_filename' is useful for URLs with redirects." ), }, "CHECK_FILESIZE_ONLY": { "default": False, "required": False, "description": ( "If True, a server's ETag and Last-Modified " "headers will not be checked to verify whether " "a download is newer than a cached item, and only " "Content-Length (filesize) will be used. This " "is useful for cases where a download always " "redirects to different mirrors, which could " "cause items to be needlessly re-downloaded. " "Defaults to False." ), }, "PKG": { "required": False, "description": ( "Local path to the pkg/dmg we'd otherwise download. " "If provided, the download is skipped and we just use " "this package or disk image." ), }, "COMPUTE_HASHES": { "required": False, "default": False, "description": ( "Local path to the pkg/dmg we'd otherwise download. " "If provided, the download is skipped and we just use " "this package or disk image." ), }, "HEADERS_TO_TEST": { "required": False, "default": ["ETag", "Last-Modified", "Content-Length"], "description": ( "Local path to the pkg/dmg we'd otherwise download. " "If provided, the download is skipped and we just use " "this package or disk image." ), }, "User_Agent": { "required": False, "description": ("User Agent Header String to use for download"), }, "download_version": { "required": False, "description": ("version of download"), }, "disable_ssl_validation": { "required": False, "default": False, "description": "Disables SSL Validation. WARNING: dangerous!", }, "download_missing_file": { "required": False, "default": True, "description": """ If the file is missing, but the metadata is present that shows it is the same file, skip download? """, }, "download_save_file": { "required": False, "default": True, "description": """ Save download file? If False, then only metadata will be saved, not the download file. This will cause problems if the download file is required for future steps. This is treated as True if `download_missing_file` is True regardless of value supplied. This is designed to be used in AWS Lambda or similar Docker use cases. Only as much RAM as consumed by `chunksize` within `download_and_hash()` plus overhead is requried because the file is never fully read into memory and never written to disk if this setting is used. """, }, } output_variables = { "pathname": {"description": "Path to the downloaded file."}, "last_modified": { "description": "last-modified header for the downloaded item." }, "etag": {"description": "etag header for the downloaded item."}, "download_changed": { "description": ( "Boolean indicating if the download has changed since the " "last time it was downloaded." ) }, "download_info": {"description": "Info from previous or current download."}, "url_downloader_summary_result": { "description": "Description of interesting results." }, } __doc__ = description def download_changed(self, header): """Check if downloaded file changed on server.""" self.output("HTTP Headers: \n{headers}".format(headers=header), 2) # get the list of headers to check headers_to_test = self.env.get("HEADERS_TO_TEST", None) self.output( "headers_to_test: {headers_to_test}".format( headers_to_test=headers_to_test ), 2, ) # get previous info to compare previous_download_info = self.get_download_info_json() if not previous_download_info: # no previous download info to check against # could check against xattr values if supported? return True # store previous download info in case we don't download again self.env["download_info"] = previous_download_info # always store last modified in case we don't download again # this wouldn't get set if `download_version` is used for download detection try: self.env["last_modified"] = previous_download_info["http_headers"][ "Last-Modified" ] except KeyError: pass self.output( "previous_download_info: \n{previous_download_info}\n".format( previous_download_info=previous_download_info ), 2, ) # store previous download_url in case we don't download again if "download_url" in previous_download_info: self.env["download_url"] = previous_download_info["download_url"] # check that previous download exits: previous_download_path = self.env.get("pathname", None) download_missing_file = self.env.get("download_missing_file", True) if not os.path.isfile(previous_download_path): # previous download doesn't exist! if download_missing_file: return True download_version = self.env.get("download_version", None) # compare versions: if download_version: self.output(download_version, 1) if "version" in previous_download_info: self.output(previous_download_info["version"], 1) if download_version != previous_download_info["version"]: self.output( "Download Changed: version does not match previous download" ) return True else: self.output( "Download UNCHANGED: version does match previous download" ) return False header_matches = 0 try: # check Content-Length: if ( headers_to_test and "Content-Length" in headers_to_test and ( int(previous_download_info["http_headers"]["Content-Length"]) != int(header.get("Content-Length")) ) ): self.output("Content-Length is different", 1) return True else: header_matches += 1 except (KeyError, TypeError) as err: self.output( "WARNING: 'Content-Length' missing. ({err_type}) {err}".format( err=err, err_type=type(err).__name__ ), 1, ) # check other headers: for test in headers_to_test: if test != "Content-Length": try: if previous_download_info["http_headers"][test] != header.get(test): self.output("{test} is different".format(test=test), 1) return True else: header_matches += 1 if test == "Last-Modified": self.env["last_modified"] = previous_download_info[ "http_headers" ][test] if test == "ETag": self.env["etag"] = previous_download_info["http_headers"][ test ] except (KeyError, TypeError) as err: self.output( "WARNING: header missing. ({err_type}) {err}".format( err=err, err_type=type(err).__name__ ), 1, ) # if no header checks work without throwing exceptions: if header_matches == 0: return True # if all above pass, then return False: # NOTE: this could think the download is unchanged when 1 header matches # but not all headers match return False def store_download_info_json(self, download_dictionary): """If file is downloaded, store info""" pathname = self.env.get("pathname") pathname_info_json = pathname + ".info.json" # put new download info into env: self.env["download_info"] = download_dictionary # https://stackoverflow.com/questions/16267767/python-writing-json-to-file with open(pathname_info_json, "w") as outfile: json.dump(download_dictionary, outfile, indent=4) # add newline at end of file: outfile.write("\n") def get_download_info_json(self): """get info from previous download""" pathname = self.env.get("pathname") pathname_info_json = pathname + ".info.json" try: with open(pathname_info_json, "r") as infile: info_json = json.load(infile) except FileNotFoundError as err: self.output( "WARNING: missing download info ({err_type})\n{err}\n".format( err=err, err_type=type(err).__name__ ), 1, ) return None return info_json def ssl_context_ignore(self): """ssl context - ignore SSL validation""" # this doesn't need to be a class method ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.output("WARNING: disabling SSL validation is insecure!!!") return ctx def ssl_context_certifi(self): """ssl context using certifi CAs""" # this doesn't need to be a class method # https://stackoverflow.com/questions/24374400/verifying-https-certificates-with-urllib-request return ssl.create_default_context(cafile=certifi.where()) def download_and_hash(self, file_save_path): """stream down file from url and calculate size & hashes""" # it is much more efficient to calculate hashes WHILE downloading # this allows the file to be read only once and never from disk # https://github.com/jgstew/bigfix_prefetch/blob/master/src/bigfix_prefetch/prefetch_from_url.py url = self.env.get("url") download_dictionary = {} hashes = None if self.env.get("COMPUTE_HASHES", None): hashes = sha1(), sha256(), md5() # chunksize seems like it could be anything # it is probably best if it is a multiple of a typical hash block_size # a larger chunksize is probably best for faster downloads # chunksize should be evenly divisible by 4096 due to 4k blocks of storage chunksize = 4096 * 100 if hashes: chunksize = max(chunksize, max(a_hash.block_size for a_hash in hashes)) size = 0 file_save = None if file_save_path: file_save = open(file_save_path, "wb") # get http headers req = Request(url) # the following may be required in some cases: user_agent_value = self.env.get("User_Agent", None) if user_agent_value: req.add_header("User-Agent", user_agent_value) ssl_context = self.ssl_context_certifi() # ssl_context = ssl.create_default_context() disable_ssl_validation = self.env.get("disable_ssl_validation", False) if disable_ssl_validation: ssl_context = self.ssl_context_ignore() # timeout of 2 hours response = urlopen(req, context=ssl_context, timeout=7200) response_headers = response.info() self.env["download_changed"] = self.download_changed(response_headers) # check if download changed from last run: if not self.env.get("download_changed", None): # Close file handle if file_save: file_save.close() # Discard the temp file if file_save_path: self.clear_zero_file(file_save_path) return None self.output("INFO: Downloading New File") # download file while True: chunk = response.read(chunksize) if not chunk: break # get size of chunk and add to existing size size += len(chunk) # add chunk to hash computations if hashes: for a_hash in hashes: a_hash.update(chunk) # save file if handler if file_save: file_save.write(chunk) # close file handler if used if file_save: file_save.close() if self.env.get("download_changed", None): # Move the new temporary download file to the pathname if file_save_path: self.move_temp_file(file_save_path) download_dictionary["file_name"] = self.env.get("filename", "") download_dictionary["file_size"] = size if hashes: download_dictionary["file_sha1"] = hashes[0].hexdigest() download_dictionary["file_sha256"] = hashes[1].hexdigest() download_dictionary["file_md5"] = hashes[2].hexdigest() download_dictionary["url"] = url # store new download url: if response.url: download_url = response.url download_dictionary["download_url"] = download_url self.env["download_url"] = download_url download_version = self.env.get( "download_version", self.env.get("version", None) ) if download_version: download_dictionary["version"] = download_version # download_dictionary['http_headers'] = response.info() try: # save http header info to dict download_dictionary["http_headers"] = {} if "content-length" in response.headers: download_dictionary["http_headers"]["Content-Length"] = int( response.headers["content-length"] ) if download_dictionary["http_headers"]["Content-Length"] != size: # should this be a halting error? self.output("WARNING: file size != content-length header", 0) if "ETag" in response.headers: download_dictionary["http_headers"]["ETag"] = response.headers["ETag"] if "Last-Modified" in response.headers: download_dictionary["http_headers"]["Last-Modified"] = response.headers[ "Last-Modified" ] except (KeyError, TypeError) as err: # probably need to handle a missing header better than this self.output( "ERROR: header issue ({err_type})\n{err}\n".format( err=err, err_type=type(err).__name__ ) ) return download_dictionary # Save last-modified and etag headers to files xattr # This is for backwards compatibility with URLDownloader try: # this can throw errors on Linux running in WSL # it might also throw errors on Linux containers self.store_headers(response.info()) except OSError as err: self.output( "ERROR xattr: ({err_type})\n{err}\n".format( err=err, err_type=type(err).__name__ ), 1, ) return download_dictionary def main(self): """Execution starts here""" # Clear and initiazize data structures self.clear_vars() # if download_missing_file is True, then download_save_file must be true download_missing_file = self.env.get("download_missing_file", True) if download_missing_file: self.env["download_save_file"] = True # Ensure existence of necessary files, directories and paths filename = self.get_filename() if filename is None: return # in some cases, the filename could have html parameters after: if "?" in filename: self.output("Removing ? and following characters from filename", 0) # this fix should be applied to URLDownloader.prefetch_filename() filename = filename.split("?", 1)[0] self.env["filename"] = filename download_dir = self.get_download_dir() self.env["pathname"] = os.path.join(download_dir, filename) # clear empty file from previous run self.clear_zero_file(self.env["pathname"]) # change headers to test if CHECK_FILESIZE_ONLY if self.env.get("CHECK_FILESIZE_ONLY", None): self.env["HEADERS_TO_TEST"] = ["Content-Length"] download_save_file = self.env.get("download_save_file", True) pathname_temporary = None # if download_save_file is false, then do not store the file being "downloaded" if download_save_file: pathname_temporary = self.create_temp_file(download_dir) # download file download_dictionary = self.download_and_hash(pathname_temporary) self.output( "download_dictionary: \n{download_dictionary}\n".format( download_dictionary=download_dictionary ) ) # clear temp file if 0 size if pathname_temporary: self.clear_zero_file(pathname_temporary) if self.env.get("download_changed", None): # store download info for checking for existing download self.store_download_info_json(download_dictionary) if download_dictionary and "http_headers" in download_dictionary: if "ETag" in download_dictionary["http_headers"]: self.env["etag"] = download_dictionary["http_headers"]["ETag"] if "Last-Modified" in download_dictionary["http_headers"]: self.env["last_modified"] = download_dictionary["http_headers"][ "Last-Modified" ] # Generate output messages and variables self.output(f"Downloaded {self.env['pathname']}") self.env["url_downloader_summary_result"] = { "summary_text": "The following new items were downloaded:", "data": {"download_path": self.env["pathname"]}, } self.output("self.env: \n{self_env}\n".format(self_env=self.env), 4) if __name__ == "__main__": PROCESSOR = URLDownloaderPython() PROCESSOR.execute_shell()