diff options
author | Antanas Uršulis <antanas.ursulis@gmail.com> | 2013-07-30 01:38:23 +0300 |
---|---|---|
committer | Antanas Uršulis <antanas.ursulis@gmail.com> | 2013-07-30 01:38:23 +0300 |
commit | 7ba5f957c082545a9eb097839fdc7eba826a7d80 (patch) | |
tree | cb10b75ef04167ea7ac127f5fb882b11fdc4cfe5 | |
parent | Add hostname to schema (diff) | |
download | log-analysis-7ba5f957c082545a9eb097839fdc7eba826a7d80.tar.gz log-analysis-7ba5f957c082545a9eb097839fdc7eba826a7d80.tar.bz2 log-analysis-7ba5f957c082545a9eb097839fdc7eba826a7d80.zip |
Implement additional metadata storage for logs received from Portage
Flags like 'pkg_failed', 'test_failed', etc. are stored once per group.
This is a discussion item, as it might make sense to store some data
once per file instead.
-rw-r--r-- | database.py | 2 | ||||
-rw-r--r-- | portage_database.py | 12 | ||||
-rw-r--r-- | portage_processor.py | 18 | ||||
-rw-r--r-- | schema_portage.sql | 10 |
4 files changed, 34 insertions, 8 deletions
diff --git a/database.py b/database.py index 79ae693..9927627 100644 --- a/database.py +++ b/database.py @@ -1,7 +1,7 @@ from contextlib import closing import MySQLdb -class DatabaseConnection: +class DatabaseConnection(object): def __init__(self, conn): self.conn = conn diff --git a/portage_database.py b/portage_database.py new file mode 100644 index 0000000..ca7831c --- /dev/null +++ b/portage_database.py @@ -0,0 +1,12 @@ +from contextlib import closing +from database import DatabaseConnection + +class PortageDatabaseConnection(DatabaseConnection): + def __init__(self, db): + super(PortageDatabaseConnection, self).__init__(db.conn) + + # TODO: consider passing these arguments around in a dictionary or kwargs + def insert_group_extra(self, group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc): + with closing(self.conn.cursor()) as c: + c.execute("insert into `groups_portage` values (%s, %s, %s, %s, %s, %s, %s, %s)", (group_id, pkg_name, matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc)) + self.conn.commit() diff --git a/portage_processor.py b/portage_processor.py index 32ca9c4..252209e 100644 --- a/portage_processor.py +++ b/portage_processor.py @@ -1,4 +1,5 @@ import os, re, StringIO, time +from portage_database import PortageDatabaseConnection class PortageProcessor: _r = { @@ -14,16 +15,17 @@ class PortageProcessor: self.storage = storage def process(self, request, source, db): + db = PortageDatabaseConnection(db) group_id = db.insert_group(source, request.group_name, 'portage', int(time.time())) - for f in request.files: - matches = 0 - pkg_failed = False - test_failed = False - collision = False - bug_assignee = 'bug-wranglers@gentoo.org' - bug_cc = '' + matches = 0 + pkg_failed = False + test_failed = False + collision = False + bug_assignee = 'bug-wranglers@gentoo.org' + bug_cc = '' + for f in request.files: # TODO: look at proper HTML generation methods: # (*) either XHTML via xml.etree # (*) or Jinja2 (is it possible to parse and generate in one pass?) @@ -74,3 +76,5 @@ class PortageProcessor: self.storage.save_file(source, f.filename, output.getvalue()) file_id = db.insert_file(os.path.join(source, f.filename), group_id) + + db.insert_group_extra(group_id, 'TODO', matches, pkg_failed, test_failed, collision, bug_assignee, bug_cc) diff --git a/schema_portage.sql b/schema_portage.sql new file mode 100644 index 0000000..4b9c80d --- /dev/null +++ b/schema_portage.sql @@ -0,0 +1,10 @@ +create table if not exists `groups_portage` ( + `id` int primary key, + `pkg_name` varchar(255) not null, + `matches` int not null, + `pkg_failed` bool not null, + `test_failed` bool not null, + `collision` bool not null, + `bug_assignee` text not null, + `bug_cc` text not null +); |