From 63e25313d785f40abff3197fe0f3a3eeaa617f7d Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 12 Jul 2012 12:48:49 -0700 Subject: Skip whirlpool digest check when unaccelerated. If multiple digests are available and whirlpool is unaccelerated, then skip it. This allows extreme performance problems like bug #425046 to be avoided whenever practical, especially for cases like stage builds where acceleration may not be available for some hashes due to minimization of dependencies. --- pym/portage/checksum.py | 20 ++++++++++++++++++++ pym/portage/manifest.py | 6 ++++-- pym/portage/package/ebuild/fetch.py | 13 +++++++------ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index bd416ac12..daf4a0cbf 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -137,8 +137,10 @@ try: except ImportError: pass +_whirlpool_unaccelerated = False if "WHIRLPOOL" not in hashfunc_map: # Bundled WHIRLPOOL implementation + _whirlpool_unaccelerated = True from portage.util.whirlpool import new as _new_whirlpool whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") @@ -197,6 +199,24 @@ def get_hash_origin(hashtype): raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") +def _filter_unaccelarated_hashes(digests): + """ + If multiple digests are available and some are unaccelerated, + then return a new dict that omits the unaccelerated ones. This + allows extreme performance problems like bug #425046 to be + avoided whenever practical, especially for cases like stage + builds where acceleration may not be available for some hashes + due to minimization of dependencies. + """ + if _whirlpool_unaccelerated and "WHIRLPOOL" in digests: + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + if len(verifiable_hash_types) > 1: + digests = dict(digests) + digests.pop("WHIRLPOOL") + + return digests + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 90324eebe..ab9186245 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -8,7 +8,8 @@ import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_filter_unaccelarated_hashes', 'portage.util:write_atomic', ) @@ -508,7 +509,8 @@ class Manifest(object): def checkFileHashes(self, ftype, fname, ignoreMissing=False): myhashes = self.fhashdict[ftype][fname] try: - ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + ok, reason = verify_all(self._getAbsname(ftype, fname), + _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])) if not ok: raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) return ok, reason diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py index b795b2826..60ed04da2 100644 --- a/pym/portage/package/ebuild/fetch.py +++ b/pym/portage/package/ebuild/fetch.py @@ -25,7 +25,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode -from portage.checksum import hashfunc_map, perform_md5, verify_all +from portage.checksum import (hashfunc_map, perform_md5, verify_all, + _filter_unaccelarated_hashes) from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -210,6 +211,7 @@ def _check_distfile(filename, digests, eout, show_errors=1): # Zero-byte distfiles are always invalid. return (False, st) else: + digests = _filter_unaccelarated_hashes(digests) if _check_digests(filename, digests, show_errors=show_errors): eout.ebegin("%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests)))) @@ -793,8 +795,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout.eend(0) continue else: - verified_ok, reason = verify_all( - myfile_path, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Previously fetched" " file: '%s'\n") % myfile, noiselevel=-1) @@ -816,7 +818,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = \ mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: digests = list(digests) digests.sort() @@ -1051,7 +1052,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... - verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) @@ -1085,7 +1087,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, else: eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: eout.ebegin("%s %s ;-)" % \ (myfile, " ".join(sorted(digests)))) -- cgit v1.2.3-65-gdbad