From df16d13ef52ef590b10af3d1ef8663afdc621a4c Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Thu, 28 Jul 2005 18:51:41 +0000 Subject: bug fixes, and updates. metadata.database now can work directly off of $PORTDIR/metadata/cache, doing eclass interpolation on the fly... so, can support people still developing via a stable portage installation. added (|iter)items to database classes, intended as faster means of pulling data. --- portage/cache/anydbm.py | 32 ++++++++++++++++++-------------- portage/cache/metadata.py | 22 +++++++++++++++++++--- portage/cache/sql_template.py | 37 ++++++++++++++++++++++++++++++++----- portage/cache/template.py | 9 ++++++++- 4 files changed, 77 insertions(+), 23 deletions(-) diff --git a/portage/cache/anydbm.py b/portage/cache/anydbm.py index c3ff03d..ab93c7b 100644 --- a/portage/cache/anydbm.py +++ b/portage/cache/anydbm.py @@ -1,7 +1,7 @@ # Copyright: 2005 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 -# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/anydbm.py,v 1.2 2005/07/13 05:51:35 ferringb Exp $ +# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/anydbm.py,v 1.3 2005/07/28 18:51:41 ferringb Exp $ anydbm_module = __import__("anydbm") import cPickle, os @@ -21,26 +21,28 @@ class database(fs_template.FsBased): default_db = '.' + default_db self._db_path = os.path.join(self.location, fs_template.gen_label(self.location, self.label)+default_db) - print "opening self._db_path=",self._db_path self.__db = None try: self.__db = anydbm_module.open(self._db_path, "w", self._perms) + + except anydbm_module.error: + # XXX handle this at some point try: self._ensure_dirs() self._ensure_dirs(self._db_path) self._ensure_access(self._db_path) - except (OSError, IOError), e: - raise cache_errors.InitializationError(self.__clas__, e) - # try again if failed - if self.__db == None: - self.__db = anydbm_module.open(self._db_path, "c", self._perms) - + raise cache_errors.InitializationError(self.__class__, e) - except anydbm_module.error, e: - # XXX handle this at some point - raise + # try again if failed + try: + if self.__db == None: + self.__db = anydbm_module.open(self._db_path, "c", self._perms) + except andbm_module.error, e: + raise cache_errors.InitializationError(self.__class__, e) + def iteritems(self): + return self.__db.iteritems() def __getitem__(self, cpv): # we override getitem because it's just a cpickling of the data handed in. @@ -61,9 +63,11 @@ class database(fs_template.FsBased): def has_key(self, cpv): return cpv in self.__db + def commit(self): pass + def __del__(self): - print "keys=",self.__db.keys() - self.__db.sync() - self.__db.close() + if "__db" in self.__dict__ and self.__db != None: + self.__db.sync() + self.__db.close() diff --git a/portage/cache/metadata.py b/portage/cache/metadata.py index dd376f6..d4fac21 100644 --- a/portage/cache/metadata.py +++ b/portage/cache/metadata.py @@ -1,11 +1,13 @@ # Copyright: 2005 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 -# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/metadata.py,v 1.2 2005/07/13 05:51:35 ferringb Exp $ +# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/metadata.py,v 1.3 2005/07/28 18:51:41 ferringb Exp $ import os, stat import fs_template import cache_errors +from portage.ebuild import eclass_cache +from template import reconstruct_eclasses # store the current key order *here*. class database(fs_template.FsBased): @@ -17,14 +19,16 @@ class database(fs_template.FsBased): def __init__(self, *args, **config): super(database,self).__init__(*args, **config) - self.location = os.path.join(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) + location = self.location + self.location = os.path.join(self.location, "metadata/cache") +# self.label.lstrip(os.path.sep).rstrip(os.path.sep)) if len(self._known_keys) > len(self.auxdbkey_order): raise Exception("less ordered keys then auxdbkeys") if not os.path.exists(self.location): self._ensure_dirs() + self.ec = eclass_cache.cache(location) def __getitem__(self, cpv): d = {} @@ -36,6 +40,12 @@ class database(fs_template.FsBased): if isinstance(e,IOError) and e.errno == 2: raise KeyError(cpv) raise cache_errors.CacheCorruption(cpv, e) + if "_eclasses_" not in d: + if "INHERITED" in d: + d["_eclasses_"] = self.ec.get_eclass_data(d["INHERITED"].split(), from_master_only=True) + del d["INHERITED"] + else: + d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"]) try: d["_mtime_"] = os.lstat(os.path.join(self.location, cpv)).st_mtime except OSError, e:raise cache_errors.CacheCorruption(cpv, e) @@ -67,6 +77,12 @@ class database(fs_template.FsBased): # # except (OSError, IOError), e: + # hack. proper solution is to make this a __setitem__ override, since template.__setitem__ + # serializes _eclasses_, then we reconstruct it. + if "_eclasses_" in d: + d["INHERITED"] = serialize_eclasses(d["_eclasses_"]).keys() + del d["_eclasses_"] + myf.writelines( [ values.get(x,"")+"\n" for x in self.auxdbkey_order] ) myf.close() self._ensure_access(fp, mtime=values["_mtime_"]) diff --git a/portage/cache/sql_template.py b/portage/cache/sql_template.py index f91b398..4c222a9 100644 --- a/portage/cache/sql_template.py +++ b/portage/cache/sql_template.py @@ -1,9 +1,10 @@ # Copyright: 2005 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 -# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/sql_template.py,v 1.2 2005/07/13 05:51:35 ferringb Exp $ +# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/sql_template.py,v 1.3 2005/07/28 18:51:41 ferringb Exp $ import template, cache_errors +from template import reconstruct_eclasses class SQLDatabase(template.database): """template class for RDBM based caches @@ -39,11 +40,11 @@ class SQLDatabase(template.database): # boolean indicating if the derived RDBMS class supports replace syntax _supports_replace = False - def __init__(self, label, auxdbkeys, **config): + def __init__(self, location, label, auxdbkeys, *args, **config): """initialize the instance. derived classes shouldn't need to override this""" - super(SQLDatabase, self).__init__(label, auxdbkeys, **config) + super(SQLDatabase, self).__init__(location, label, auxdbkeys, *args, **config) config.setdefault("host","127.0.0.1") config.setdefault("autocommit", self.autocommits) @@ -132,8 +133,9 @@ class SQLDatabase(template.database): def __del__(self): # just to be safe. - self.commit() - self.db.close() + if "db" in self.__dict__ and self.db != None: + self.commit() + self.db.close() def _setitem(self, cpv, values): @@ -220,6 +222,31 @@ class SQLDatabase(template.database): for x in self.con.fetchall(): yield x[0] + def iteritems(self): + try: self.con.execute("SELECT cpv, key, value FROM %s NATURAL JOIN %s " + "WHERE label=%s" % (self.SCHEMA_PACKAGE_NAME, self.SCHEMA_VALUES_NAME, + self.label)) + except self._BaseError, e: + raise cache_errors.CacheCorruption(self, cpv, e) + + oldcpv = None + l = [] + for x, y, v in self.con.fetchall(): + if oldcpv != x: + if oldcpv != None: + d = dict(l) + if "_eclasses_" in d: + d["_eclasses_"] = reconstruct_eclasses(oldcpv, d["_eclasses_"]) + yield cpv, d + l.clear() + oldcpv = x + l.append((y,v)) + if oldcpv != None: + d = dict(l) + if "_eclasses_" in d: + d["_eclasses_"] = reconstruct_eclasses(oldcpv, d["_eclasses_"]) + yield cpv, d + def commit(self): self.db.commit() diff --git a/portage/cache/template.py b/portage/cache/template.py index 6d087e5..f27ed03 100644 --- a/portage/cache/template.py +++ b/portage/cache/template.py @@ -1,7 +1,7 @@ # Copyright: 2005 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 -# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/template.py,v 1.3 2005/07/28 05:44:42 ferringb Exp $ +# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/cache/template.py,v 1.4 2005/07/28 18:51:41 ferringb Exp $ import cache_errors, copy @@ -91,6 +91,13 @@ class database(object): def iterkeys(self): raise NotImplementedError + def iteritems(self): + for x in self.iterkeys(): + yield x, self[x] + + def items(self): + return list(self.iteritems()) + def sync(self, rate=0): self.sync_rate = rate if(rate == 0): -- cgit v1.2.3-65-gdbad