Added version check plugin

3 files changed, 150 insertions, 0 deletions

diff --git a/grumpy/vdb/__init__.py b/grumpy/vdb/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/grumpy/vdb/__init__.py
diff --git a/grumpy/vdb/pypi.py b/grumpy/vdb/pypi.py
new file mode 100644
index 0000000..2e27059
--- /dev/null
+++ b/grumpy/vdb/pypi.py
@@ -0,0 +1,45 @@
+from lxml.html import fromstring
+import urllib
+
+class PyPi(object):
+    # TODO: Shall we use some kind of yaml or json based format instead?
+    # Or move this data into DB?
+    pkgs = {
+        # Package in Portage            :  Package in PyPi
+        'dev-python/flask'              : 'Flask',
+        'dev-python/flask-openid'       : 'Flask-OpenID',
+        'dev-python/flask-sqlalchemy'   : 'Flask-SQLAlchemy',
+        'dev-python/mako'               : 'Mako',
+        'dev-python/sphinx'             : 'Sphinx',
+        'dev-python/virtualenv'         : 'virtualenv',
+        'dev-python/vobject'            : 'vobject',
+        'dev-python/werkzeug'           : 'Werkzeug',
+        'dev-python/xlwt'               : 'xlwt',
+        'dev-python/yolk'               : 'yolk',
+        'dev-python/yolk-portage'       : 'yolk-portage',
+    }
+
+    # Url for fetching version information
+    url = 'http://pypi.python.org/pypi?:action=index'
+
+    def __init__(self):
+        pass
+
+    def fetch_and_parse_all(self):
+        """Download and parse package version information."""
+
+        items = {}
+        #f = urllib.urlopen(self.url)
+        f = open('utils/pypi.html')
+        if f:
+            data = fromstring(f.read()).cssselect('table.list')[0]
+            for row in data.getchildren():
+                val = row[0].getchildren()
+                if len(val) > 0:
+                    pkg = val[0].text.encode('utf-8').split('\xc2\xa0')
+                    if len(pkg) == 2:
+                        if pkg[0] not in items.keys():
+                            items[pkg[0]] = []
+                        items[pkg[0]].append(pkg[1])
+        f.close()
+        return items
diff --git a/utils/version_check.py b/utils/version_check.py
new file mode 100644
index 0000000..20d1caa
--- /dev/null
+++ b/utils/version_check.py
@@ -0,0 +1,105 @@
+#! /usr/bin/env python
+import os, sys
+from datetime import datetime
+from lxml.html import fromstring
+from optparse import OptionParser
+
+path = os.path.join(os.path.dirname(__file__), os.path.pardir)
+sys.path.insert(0, path)
+del path
+
+from grumpy import app
+from grumpy.models import db, Package, PkgIssue, Setting
+from grumpy.utils import compare_version
+
+PLUGIN_NAME='ver_bumper::'
+
+def gc_collect(timestamp):
+    """Remove old QA issues from database returning number of rows deleted."""
+    db.session.expire_all()
+    print ("DEBUG: Deleted %d old issues." % PkgIssue.query \
+                        .filter_by(plugin=PLUGIN_NAME) \
+                        .filter(PkgIssue.created_on < timestamp).delete(False))
+
+def insert_issues(invalid):
+    """Insert QA issues into db."""
+    if 'maintainer-needed@gentoo.org' in invalid:
+        h = Developer.query.filter_by(email='maintainer-needed@gentoo.org').first()
+        for pkg in h.packages:
+            pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'maintainer-needed'))
+        invalid.remove('maintainer-needed@gentoo.org')
+    for dev in invalid:
+        d = Developer.query.filter_by(email=dev).first()
+        for pkg in d.packages:
+            pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'unknown-dev', \
+                      'Maintainer %s listed in metadata.xml is not as ' \
+                      'an active Gentoo developer.' % dev))
+    db.session.commit()
+
+if __name__ == '__main__':
+    from grumpy.vdb.pypi import PyPi
+    parser = OptionParser(usage="usage: %prog [options] CONFFILE")
+    (opts, args) = parser.parse_args()
+    if len(args) != 1:
+        parser.error("provide path to configuration file as first argument")
+        sys.exit(1)
+    # ...
+    pypi = PyPi()
+    items = pypi.fetch_and_parse_all()
+    # Gather package versions
+    updates = {}
+    missing = []
+    for p in pypi.pkgs:
+        pyp = pypi.pkgs[p]
+        if pyp not in items.keys():
+            missing.append([p, pyp])
+        else:
+            updates[p] = items[pyp]
+    PLUGIN_NAME = PLUGIN_NAME + pypi.__class__.__name__
+    with app.test_request_context():
+        timestamp = datetime.now()
+        app.config.from_pyfile(args[0])
+        try:
+            # Delete existing issues
+            print ("DEBUG: Deleted %d old issues." % PkgIssue.query \
+                        .filter_by(plugin=PLUGIN_NAME) \
+                        .filter(PkgIssue.created_on < timestamp).delete(False))
+            db.session.commit()
+
+            # File 'missing' package mapping issues
+            for item in missing:
+                p = Package.query.filter_by(key=item).first()
+                if not p:
+                    print "DEBUG: package '%s' is missing from upstream." % item
+                    continue
+                p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \
+                    'missing-upstream', \
+                    'No package "%s" found in upstream: %s' % \
+                        (item, pypi.__class__.__name__)))
+            # Run version checks
+            # TODO: Figure out slotted ebuilds
+            for pkg, ver in updates.iteritems():
+                p = Package.query.filter_by(key=pkg).first()
+                if not p:
+                    print "DEBUG: package '%s' is missing from portage." % item
+                    continue
+                # We need to sort pkg versions
+                ver = sorted(ver, compare_version, reverse=True)
+                res = compare_version(ver[0], p.versions[0])
+                if res == -1:
+                    p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \
+                    'version-mismatch', 'Package "%s" has newer version in ' \
+                    'Portage than on "%s" upstream' % \
+                    (pkg, pypi.__class__.__name__)))
+                elif res == 1:
+                    p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \
+                    'version-bump', 'Package "%s" has newer version "%s" '
+                    'available in "%s" upstream ' % \
+                    (pkg, p.versions[0], pypi.__class__.__name__)))
+                db.session.commit()
+            # Update settings and add info about last run..
+            Setting.query.filter_by(name=PLUGIN_NAME).delete(False)
+            db.session.add(Setting(PLUGIN_NAME, str(timestamp)))
+            db.session.commit()
+        except Exception:
+            db.session.rollback()