diff options
author | 2014-05-10 03:49:12 +0200 | |
---|---|---|
committer | 2014-05-10 03:49:12 +0200 | |
commit | e342f62e8cc2265b6085700de85c8a0eea2c0b98 (patch) | |
tree | 0e0d893241e6f28e68918c9951f8570435377fc4 | |
parent | remove gs-ctan as Gentoo already has good support for CTAN packages (diff) | |
download | g-sorcery-e342f62e8cc2265b6085700de85c8a0eea2c0b98.tar.gz g-sorcery-e342f62e8cc2265b6085700de85c8a0eea2c0b98.tar.bz2 g-sorcery-e342f62e8cc2265b6085700de85c8a0eea2c0b98.zip |
gs-pypi moved to another repo
-rw-r--r-- | bin/gs-pypi | 3 | ||||
-rwxr-xr-x | bin/gs-pypi-generate-db | 18 | ||||
-rw-r--r-- | docs/Makefile | 2 | ||||
-rw-r--r-- | docs/gs-pypi.8 | 153 | ||||
-rw-r--r-- | docs/gs-pypi.8.rst | 130 | ||||
-rw-r--r-- | gs-pypi-overlays.xml | 14 | ||||
-rw-r--r-- | gs-pypi.json | 46 | ||||
-rw-r--r-- | gs_pypi/__init__.py | 2 | ||||
-rw-r--r-- | gs_pypi/backend.py | 35 | ||||
-rw-r--r-- | gs_pypi/data/gs-pypi.eclass | 15 | ||||
-rw-r--r-- | gs_pypi/ebuild.py | 67 | ||||
-rw-r--r-- | gs_pypi/gs_pypi_generate_db.py | 60 | ||||
-rw-r--r-- | gs_pypi/pypi_db.py | 289 | ||||
-rw-r--r-- | setup.py | 12 |
14 files changed, 5 insertions, 841 deletions
diff --git a/bin/gs-pypi b/bin/gs-pypi deleted file mode 100644 index fbaabd4..0000000 --- a/bin/gs-pypi +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -g-sorcery gs-pypi $@ diff --git a/bin/gs-pypi-generate-db b/bin/gs-pypi-generate-db deleted file mode 100755 index 81710ec..0000000 --- a/bin/gs-pypi-generate-db +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - gs-pypi-generate-db - ~~~~~~~~~~~~~~~~~~~ - - PyPI database generation - - :copyright: (c) 2013 by Jauhien Piatlicki - :license: GPL-2, see LICENSE for more details. -""" - -import sys -from gs_pypi import gs_pypi_generate_db - -if __name__ == "__main__": - sys.exit(gs_pypi_generate_db.main()) diff --git a/docs/Makefile b/docs/Makefile index 5c25523..6c47097 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,7 +1,7 @@ HTML_SOURCES=developer_instructions HTML_DOCS=$(HTML_SOURCES:=.html) -MAN_SOURCES=g-sorcery g-sorcery.cfg gs-pypi +MAN_SOURCES=g-sorcery g-sorcery.cfg MANS=$(MAN_SOURCES:=.8) RST2HTML=rst2html.py diff --git a/docs/gs-pypi.8 b/docs/gs-pypi.8 deleted file mode 100644 index 7536605..0000000 --- a/docs/gs-pypi.8 +++ /dev/null @@ -1,153 +0,0 @@ -.\" Man page generated from reStructuredText. -. -.TH GS-PYPI 8 "2013-08-04" "0.1" "g-sorcery" -.SH NAME -gs-pypi \- manage overlays for PYPI repository -. -.nr rst2man-indent-level 0 -. -.de1 rstReportMargin -\\$1 \\n[an-margin] -level \\n[rst2man-indent-level] -level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] -- -\\n[rst2man-indent0] -\\n[rst2man-indent1] -\\n[rst2man-indent2] -.. -.de1 INDENT -.\" .rstReportMargin pre: -. RS \\$1 -. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] -. nr rst2man-indent-level +1 -.\" .rstReportMargin post: -.. -.de UNINDENT -. RE -.\" indent \\n[an-margin] -.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] -.nr rst2man-indent-level -1 -.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] -.in \\n[rst2man-indent\\n[rst2man-indent-level]]u -.. -.SH SYNOPSIS -.sp -\fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBsync\fP -.sp -\fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBlist\fP -.sp -\fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\fP \fIPACKAGE\fP -.sp -\fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBinstall\fP \fIPACKAGE\fP -.sp -\fBgs\-pypi\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\-tree\fP [\fB\-d\fP] -.SH DESCRIPTION -.sp -\fBgs\-pypi\fP is an ebuild generator for PYPI PYTHON repository. -.sp -There are two ways of using \fBgs\-pypi\fP: -.INDENT 0.0 -.INDENT 3.5 -.INDENT 0.0 -.IP \(bu 2 -use it with \fBlayman\fP -.sp -In this case all you need to do is install \fBlayman\-9999\fP and \fBg\-sorcery\fP. -Then you should just run \fIlayman \-L\fP as -root and find an overlay you want. Type of overlay will be -displayed as \fIg\-sorcery\fP. Then you add this overlay as -usual. It\(aqs all you need to do and it\(aqs the recommended way of -using \fBgs\-pypi\fP. -.IP \(bu 2 -use it as stand\-alone tool -.sp -In this case you should create an overlay (see \fBportage\fP documentation), sync it and populate -it with one or more ebuilds. Then ebuilds could be installed by emerge or by \fBgs\-pypi\fP tool. -.UNINDENT -.UNINDENT -.UNINDENT -.SH OPTIONS -.INDENT 0.0 -.TP -.B \fB\-\-overlay\fP \fIOVERLAY\fP, \fB\-o\fP \fIOVERLAY\fP -Overlay directory. This option is mandatory if there is no -\fBdefault_overlay\fP entry in a backend config. -.TP -.B \fB\-\-repository\fP \fIREPO\fP, \fB\-r\fP \fIREPO\fP -Repository name. This option is not mandatory. If present should be \fBctan\fP. -.UNINDENT -.SH COMMANDS -.INDENT 0.0 -.TP -.B \fBsync\fP -Synchronize a repository database. -.TP -.B \fBlist\fP -List packages available in a repository. -.TP -.B \fBgenerate\fP -Generate a given ebuild and all its dependencies. -.TP -.B \fBinstall\fP -Generate and install an ebuild using your package mangler. -.TP -.B \fBgenerate\-tree\fP -Generate entire overlay structure. Without option \fB\-d\fP after -this command sources are not fetched during generation and there -are no entries for them in Manifest files. -.UNINDENT -.SH FILES -.INDENT 0.0 -.TP -.B \fB/etc/g\-sorcery/gs\-pypi.json\fP -Backend config. -.TP -.B \fB/etc/layman/overlays/gs\-pypi\-overlays.xml\fP -List of available repositories. -.UNINDENT -.SH EXAMPLES -.INDENT 0.0 -.TP -.B Using gs\-pypi with layman -Execute -.sp -\fBlayman \-L\fP -.sp -If you see there a \fBpypi\fP overlay then anything should work. -.sp -\fBlayman \-a pypi\fP -.sp -Emerge any package from it using \fBemerge\fP. -.TP -.B Generating user ebuilds in user overlay -Create new user overlay. Run -.sp -\fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBsync\fP -.sp -List packages: -.sp -\fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBlist\fP -.sp -Install any package you want: -.sp -\fBgs\-pypi \-o\fP \fIOVERLAY_DIRECTORY\fP \fB\-r ctan\fP \fBinstall\fP \fIPACKAGE\fP -.sp -Note, that if you call \fBgenerate\-tree\fP command your overlay -will be wiped and overlay tree for a given repository will be generated. Be careful! -.UNINDENT -.SH NOTES -.INDENT 0.0 -.IP 1. 3 -At the moment the only package mangler \fBgs\-pypi\fP supports is \fBportage\fP. -.UNINDENT -.SH SEE ALSO -.sp -\fBgs\-elpa\fP(8), \fBg\-sorcery.cfg\fP(8), \fBportage\fP(5), \fBemerge\fP(1), \fBlayman\fP(8) -.SH AUTHOR -Written by Jauhien Piatlicki <piatlicki@gmail.com>. GSoC idea -and mentorship by Rafael Martins. Lots of help and improvements -by Brian Dolbec. -.SH COPYRIGHT -Copyright (c) 2013 Jauhien Piatlicki, License: GPL-2 -.\" Generated by docutils manpage writer. -. diff --git a/docs/gs-pypi.8.rst b/docs/gs-pypi.8.rst deleted file mode 100644 index 92031cc..0000000 --- a/docs/gs-pypi.8.rst +++ /dev/null @@ -1,130 +0,0 @@ -======= -gs-pypi -======= - ------------------------------------ -manage overlays for PYPI repository ------------------------------------ - -:Author: Written by Jauhien Piatlicki <piatlicki@gmail.com>. GSoC idea - and mentorship by Rafael Martins. Lots of help and improvements - by Brian Dolbec. -:Date: 2013-08-04 -:Copyright: Copyright (c) 2013 Jauhien Piatlicki, License: GPL-2 -:Version: 0.1 -:Manual section: 8 -:Manual group: g-sorcery - - -SYNOPSIS -======== - -**gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **sync** - -**gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **list** - -**gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **generate** *PACKAGE* - -**gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **install** *PACKAGE* - -**gs-pypi** **-o** *OVERLAY* [**-r** *REPO*] **generate-tree** [**-d**] - -DESCRIPTION -=========== - -**gs-pypi** is an ebuild generator for PYPI PYTHON repository. - -There are two ways of using **gs-pypi**: - - * use it with **layman** - - In this case all you need to do is install **layman-9999** and **g-sorcery**. - Then you should just run `layman -L` as - root and find an overlay you want. Type of overlay will be - displayed as *g-sorcery*. Then you add this overlay as - usual. It's all you need to do and it's the recommended way of - using **gs-pypi**. - - * use it as stand-alone tool - - In this case you should create an overlay (see **portage** documentation), sync it and populate - it with one or more ebuilds. Then ebuilds could be installed by emerge or by **gs-pypi** tool. - - -OPTIONS -======= - -**--overlay** *OVERLAY*, **-o** *OVERLAY* - Overlay directory. This option is mandatory if there is no - **default_overlay** entry in a backend config. - -**--repository** *REPO*, **-r** *REPO* - Repository name. This option is not mandatory. If present should be **ctan**. - -COMMANDS -======== - -**sync** - Synchronize a repository database. - -**list** - List packages available in a repository. - -**generate** - Generate a given ebuild and all its dependencies. - -**install** - Generate and install an ebuild using your package mangler. - -**generate-tree** - Generate entire overlay structure. Without option **-d** after - this command sources are not fetched during generation and there - are no entries for them in Manifest files. - -FILES -===== -**/etc/g-sorcery/gs-pypi.json** - Backend config. - -**/etc/layman/overlays/gs-pypi-overlays.xml** - List of available repositories. - -EXAMPLES -======== - -Using gs-pypi with layman - Execute - - **layman -L** - - If you see there a **pypi** overlay then anything should work. - - **layman -a pypi** - - Emerge any package from it using **emerge**. - -Generating user ebuilds in user overlay - Create new user overlay. Run - - **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **sync** - - List packages: - - **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **list** - - Install any package you want: - - **gs-pypi -o** *OVERLAY_DIRECTORY* **-r ctan** **install** *PACKAGE* - - Note, that if you call **generate-tree** command your overlay - will be wiped and overlay tree for a given repository will be generated. Be careful! - -NOTES -===== - -1. At the moment the only package mangler **gs-pypi** supports is **portage**. - -SEE ALSO -======== - -**gs-elpa**\(8), **g-sorcery.cfg**\(8), **portage**\(5), **emerge**\(1), **layman**\(8) diff --git a/gs-pypi-overlays.xml b/gs-pypi-overlays.xml deleted file mode 100644 index 6aba9e7..0000000 --- a/gs-pypi-overlays.xml +++ /dev/null @@ -1,14 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE repositories SYSTEM "/dtd/repositories.dtd"> -<repositories xmlns="" version="1.0"> -<repo quality="experimental" status="unofficial"> - <name>pypi</name> - <description>python packages</description> - <homepage>https://pypi.python.org/pypi</homepage> - <owner> - <email>piatlicki@gmail.com</email> - <name>Jauhien Piatlicki</name> - </owner> - <source type="g-sorcery">gs-pypi pypi</source> -</repo> -</repositories> diff --git a/gs-pypi.json b/gs-pypi.json deleted file mode 100644 index 7c239a7..0000000 --- a/gs-pypi.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "backend": "gs-pypi", - "package": "gs_pypi", - "repositories": { - "pypi": { - "repo_uri": "https://pypi.python.org/", - "db_uri": "https://github.com/jauhien/gs-pypi-db/archive/master.tar.gz" - } - }, - "common_config": { - "licenses": { - "Academic Free License (AFL)": "AFL-3.0", - "Aladdin Free Public License (AFPL)": "Aladdin", - "Apache Software License": "Apache-2.0", - "Artistic License": "Artistic", - "BSD License": "BSD", - "CC0 1.0 Universal (CC0 1.0) Public Domain Dedication": "CC0-1.0", - "Common Public License": "CPL-1.0", - "GNU Affero General Public License v3": "AGPL-3", - "GNU Affero General Public License v3 or later (AGPLv3+)": "AGPL-3", - "GNU Free Documentation License (FDL)": "FDL-1.1+", - "GNU General Public License (GPL)": "GPL-1+", - "GNU General Public License v2 (GPLv2)": "GPL-2", - "GNU General Public License v2 or later (GPLv2+)": "GPL-2+", - "GNU General Public License v3 (GPLv3)": "GPL-3", - "GNU General Public License v3 or later (GPLv3+)": "GPL-3+", - "GNU Lesser General Public License v2 (LGPLv2)": "LGPL-2", - "GNU Lesser General Public License v2 or later (LGPLv2+)": "LGPL-2+", - "GNU Lesser General Public License v3 (LGPLv3)": "LGPL-3", - "GNU Lesser General Public License v3 or later (LGPLv3+)": "LGPL-3+", - "GNU Library or Lesser General Public License (LGPL)": "LGPL-2+", - "ISC License (ISCL)": "ISC", - "MIT License": "MIT", - "Mozilla Public License 1.0 (MPL)": "MPL-1.0", - "Mozilla Public License 1.1 (MPL 1.1)": "MPL-1.1", - "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", - "Public Domain": "public-domain", - "Python License (CNRI Python License)": "CNRI", - "Python Software Foundation License": "PYTHON", - "Repoze Public License": "repoze", - "W3C License": "W3C", - "Zope Public License": "ZPL", - "zlib/libpng License": "ZLIB" - } - } -} diff --git a/gs_pypi/__init__.py b/gs_pypi/__init__.py deleted file mode 100644 index cf529d7..0000000 --- a/gs_pypi/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python - diff --git a/gs_pypi/backend.py b/gs_pypi/backend.py deleted file mode 100644 index b9e9988..0000000 --- a/gs_pypi/backend.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - backend.py - ~~~~~~~~~~ - - PyPI backend - - :copyright: (c) 2013 by Jauhien Piatlicki - :license: GPL-2, see LICENSE for more details. -""" - -import os - -from g_sorcery.backend import Backend -from g_sorcery.metadata import MetadataGenerator -from g_sorcery.eclass import EclassGenerator -from g_sorcery.fileutils import get_pkgpath - -from .pypi_db import PypiDBGenerator -from .ebuild import PypiEbuildWithoutDigestGenerator, PypiEbuildWithDigestGenerator - - -class PypiEclassGenerator(EclassGenerator): - """ - Implementation of eclass generator. Only specifies a data directory. - """ - def __init__(self): - super(PypiEclassGenerator, self).__init__(os.path.join(get_pkgpath(__file__), 'data')) - - -instance = Backend(PypiDBGenerator, - PypiEbuildWithDigestGenerator, PypiEbuildWithoutDigestGenerator, - PypiEclassGenerator, MetadataGenerator, sync_db=True) diff --git a/gs_pypi/data/gs-pypi.eclass b/gs_pypi/data/gs-pypi.eclass deleted file mode 100644 index c35796c..0000000 --- a/gs_pypi/data/gs-pypi.eclass +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 1999-2013 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Header: $ -# automatically generated by gs-pypi -# please do not edit this file -# -# Original Author: Jauhien Piatlicki <piatlicki@gmail.com> -# Purpose: support installation of python packages from PyPI repo -# -# Bugs to piatlicki@gmail.com -# -# @ECLASS: gs-pypi.eclass -# - -inherit distutils-r1 g-sorcery diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py deleted file mode 100644 index 82024c3..0000000 --- a/gs_pypi/ebuild.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - ebuild.py - ~~~~~~~~~ - - ebuild generation - - :copyright: (c) 2013 by Jauhien Piatlicki - :license: GPL-2, see LICENSE for more details. -""" - -import collections -import os - -from g_sorcery.ebuild import DefaultEbuildGenerator - -Layout = collections.namedtuple("Layout", - ["vars_before_inherit", "inherit", "vars_after_description", "vars_after_keywords"]) - - -class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator): - """ - Implementation of ebuild generator without sources digesting. - """ - def __init__(self, package_db): - - vars_before_inherit = \ - ["realname", "realversion", - {"name" : "repo_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/'}, - {"name" : "sourcefile", "value" : '${REALNAME}-${REALVERSION}.tar.gz'}, {"name" : "python_compat", "raw" : True}] - - inherit = ["gs-pypi"] - - vars_after_description = \ - ["homepage", "license"] - - vars_after_keywords = \ - [] - - layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords) - - super(PypiEbuildWithoutDigestGenerator, self).__init__(package_db, layout) - -class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator): - """ - Implementation of ebuild generator with sources digesting. - """ - def __init__(self, package_db): - - vars_before_inherit = \ - ["realname", "realversion", - {"name" : "digest_sources", "value" : "yes"}, {"name" : "python_compat", "raw" : True}] - - inherit = ["gs-pypi"] - - vars_after_description = \ - ["homepage", "license", - {"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/${REALNAME}-${REALVERSION}.tar.gz'}] - - vars_after_keywords = \ - [] - - layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords) - - super(PypiEbuildWithDigestGenerator, self).__init__(package_db, layout) diff --git a/gs_pypi/gs_pypi_generate_db.py b/gs_pypi/gs_pypi_generate_db.py deleted file mode 100644 index a59b5e3..0000000 --- a/gs_pypi/gs_pypi_generate_db.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - pypi_db.py - ~~~~~~~~~~ - - PyPI database generation - - :copyright: (c) 2013 by Jauhien Piatlicki - :license: GPL-2, see LICENSE for more details. -""" - -import os -import sys - -from g_sorcery.compatibility import TemporaryDirectory -from g_sorcery.exceptions import FileJSONError -from g_sorcery.fileutils import copy_all, FileJSON -from g_sorcery.logger import Logger - -from .pypi_db import PypiDBGenerator - -def main(): - - if len(sys.argv) < 2: - print("Usage: ") - print("gs-pypi-generate-db db_name") - return -1 - - logger = Logger() - cfg_path = None - for path in '.', '~', '/etc/g-sorcery': - current = os.path.join(path, "gs-pypi.json") - if (os.path.isfile(current)): - cfg_path = path - break - if not cfg_path: - logger.error('no config file for gs-pypi backend\n') - return -1 - cfg_f = FileJSON(cfg_path, "gs-pypi.json", ['package']) - try: - config = cfg_f.read() - except FileJSONError as e: - logger.error('error loading config file for gs-pypi: ' + str(e) + '\n') - return -1 - - generator = PypiDBGenerator() - db_name = sys.argv[1] - temp_dir = TemporaryDirectory() - pkg_db = generator(temp_dir.name, "pypi", config=config["repositories"]["pypi"], common_config=config["common_config"]) - if os.path.exists(db_name): - os.system('rm -rf ' + db_name + '/*') - else: - os.mkdir(db_name) - copy_all(os.path.join(temp_dir.name, "pypi/db"), db_name) - os.system('tar cvzf ' + db_name + '.tar.gz ' + db_name) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py deleted file mode 100644 index 2b201c6..0000000 --- a/gs_pypi/pypi_db.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - pypi_db.py - ~~~~~~~~~~ - - PyPI package database - - :copyright: (c) 2013 by Jauhien Piatlicki - :license: GPL-2, see LICENSE for more details. -""" - -import datetime -import re -import time - -import bs4 - -from g_sorcery.exceptions import DownloadingError -from g_sorcery.g_collections import Package, serializable_elist -from g_sorcery.package_db import DBGenerator - -class PypiDBGenerator(DBGenerator): - """ - Implementation of database generator for PYPI backend. - """ - def get_download_uries(self, common_config, config): - """ - Get URI of packages index. - """ - self.repo_uri = config["repo_uri"] - return [{"uri": self.repo_uri + "pypi?%3Aaction=index", "output": "packages"}] - - def parse_data(self, data_f): - """ - Download and parse packages index. Then download and parse pages for all packages. - """ - soup = bs4.BeautifulSoup(data_f.read()) - packages = soup.table - data = {} - data["index"] = {} - - pkg_uries = [] - - for entry in packages.find_all("tr")[1:-1]: - package, description = entry.find_all("td") - - if description.contents: - description = description.contents[0] - else: - description = "" - package, version = package.a["href"].split("/")[2:] - data["index"][(package, version)] = description - pkg_uries.append({"uri": self.repo_uri + "pypi/" + package + "/" + version, - "parser": self.parse_package_page, - "output": package + "-" + version}) - entry.decompose() - - packages.decompose() - soup.decompose() - - pkg_uries = self.decode_download_uries(pkg_uries) - for uri in pkg_uries: - attempts = 0 - while True: - try: - attempts += 1 - self.process_uri(uri, data) - except DownloadingError as error: - print(str(error)) - time.sleep(5) - if attempts < 100: - continue - break - - return data - - def parse_package_page(self, data_f): - """ - Parse package page. - """ - soup = bs4.BeautifulSoup(data_f.read()) - data = {} - data["files"] = [] - data["info"] = {} - try: - for table in soup("table", class_ = "list")[-1:]: - if not "File" in table("th")[0].string: - continue - - for entry in table("tr")[1:-1]: - fields = entry("td") - - FILE = 0 - URL = 0 - MD5 = 1 - - TYPE = 1 - PYVERSION = 2 - UPLOADED = 3 - SIZE = 4 - - file_inf = fields[FILE]("a")[0]["href"].split("#") - file_url = file_inf[URL] - file_md5 = file_inf[MD5][4:] - - file_type = fields[TYPE].string - file_pyversion = fields[PYVERSION].string - file_uploaded = fields[UPLOADED].string - file_size = fields[SIZE].string - - data["files"].append({"url": file_url, - "md5": file_md5, - "type": file_type, - "pyversion": file_pyversion, - "uploaded": file_uploaded, - "size": file_size}) - entry.decompose() - table.decompose() - - uls = soup("ul", class_ = "nodot") - if uls: - if "Downloads (All Versions):" in uls[0]("strong")[0].string: - ul = uls[1] - else: - ul = uls[0] - - for entry in ul.contents: - if not hasattr(entry, "name") or entry.name != "li": - continue - entry_name = entry("strong")[0].string - if not entry_name: - continue - - if entry_name == "Categories": - data["info"][entry_name] = {} - for cat_entry in entry("a"): - cat_data = cat_entry.string.split(" :: ") - if not cat_data[0] in data["info"][entry_name]: - data["info"][entry_name][cat_data[0]] = cat_data[1:] - else: - data["info"][entry_name][cat_data[0]].extend(cat_data[1:]) - continue - - if entry("span"): - data["info"][entry_name] = entry("span")[0].string - continue - - if entry("a"): - data["info"][entry_name] = entry("a")[0]["href"] - continue - entry.decompose() - ul.decompose() - - except Exception as error: - print("There was an error during parsing: " + str(error)) - print("Ignoring this package.") - data = {} - data["files"] = [] - data["info"] = {} - - soup.decompose() - return data - - def process_data(self, pkg_db, data, common_config, config): - """ - Process parsed package data. - """ - category = "dev-python" - pkg_db.add_category(category) - - #todo: write filter functions - allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ - set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, - ['+', '_', '-']))) - - allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \ - set(range(ord('0'), ord('9') + 1)) | set(list(map(ord, - ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ',']))) - - now = datetime.datetime.now() - pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day) - - for (package, version), description in data["packages"]["index"].items(): - - pkg = package + "-" + version - if not pkg in data["packages"]: - continue - - pkg_data = data["packages"][pkg] - - if not pkg_data["files"] and not pkg_data["info"]: - continue - - files_src_uri = "" - md5 = "" - if pkg_data["files"]: - for file_entry in pkg_data["files"]: - if file_entry["type"] == "\n Source\n ": - files_src_uri = file_entry["url"] - md5 = file_entry["md5"] - break - - download_url = "" - info = pkg_data["info"] - if info: - if "Download URL:" in info: - download_url = pkg_data["info"]["Download URL:"] - - if download_url: - source_uri = download_url #todo: find how to define src_uri - else: - source_uri = files_src_uri - - if not source_uri: - continue - - homepage = "" - pkg_license = "" - py_versions = [] - if info: - if "Home Page:" in info: - homepage = info["Home Page:"] - categories = {} - if "Categories" in info: - categories = info["Categories"] - - if 'Programming Language' in categories: - for entry in categories['Programming Language']: - if entry == '2': - py_versions.extend(['2_6', '2_7']) - elif entry == '3': - py_versions.extend(['3_2', '3_3']) - elif entry == '2.6': - py_versions.extend(['2_6']) - elif entry == '2.7': - py_versions.extend(['2_7']) - elif entry == '3.2': - py_versions.extend(['3_2']) - elif entry == '3.3': - py_versions.extend(['3_3']) - - if "License" in categories: - pkg_license = categories["License"][-1] - pkg_license = self.convert([common_config, config], "licenses", pkg_license) - - if not py_versions: - py_versions = ['2_6', '2_7', '3_2', '3_3'] - if len(py_versions) == 1: - python_compat = 'python' + py_versions[0] - else: - python_compat = '( python{' + py_versions[0] - for ver in py_versions[1:]: - python_compat += ',' + ver - python_compat += '} )' - - filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg]) - description = "".join([x for x in description if ord(x) in allowed_ords_desc]) - filtered_version = version - match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", - filtered_version) - if not match_object: - filtered_version = pseudoversion - - dependencies = serializable_elist(separator="\n\t") - eclasses = ['g-sorcery', 'gs-pypi'] - maintainer = [{'email' : 'piatlicki@gmail.com', - 'name' : 'Jauhien Piatlicki'}] - - ebuild_data = {} - ebuild_data["realname"] = package - ebuild_data["realversion"] = version - - ebuild_data["description"] = description - ebuild_data["longdescription"] = description - ebuild_data["dependencies"] = dependencies - ebuild_data["eclasses"] = eclasses - ebuild_data["maintainer"] = maintainer - - ebuild_data["homepage"] = homepage - ebuild_data["license"] = pkg_license - ebuild_data["source_uri"] = source_uri - ebuild_data["md5"] = md5 - ebuild_data["python_compat"] = python_compat - - ebuild_data["info"] = info - - pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data) @@ -7,13 +7,9 @@ setup(name = 'g-sorcery', description = 'g-sorcery framework for automated ebuild generators', author = 'Jauhien Piatlicki', author_email = 'jauhien@gentoo.org', - packages = ['g_sorcery', 'gs_db_tool', 'gs_pypi'], - package_data = {'g_sorcery': ['data/*'], - 'gs_pypi': ['data/*']}, - scripts = ['bin/g-sorcery', 'bin/gs-db-tool', - 'bin/gs-pypi-generate-db', 'bin/gs-pypi'], - data_files = [('/etc/g-sorcery/', ['gs-pypi.json']), - ('/etc/g-sorcery/', ['g-sorcery.cfg']), - ('/etc/layman/overlays/', ['gs-pypi-overlays.xml'])], + packages = ['g_sorcery', 'gs_db_tool'], + package_data = {'g_sorcery': ['data/*']}, + scripts = ['bin/g-sorcery', 'bin/gs-db-tool'], + data_files = [('/etc/g-sorcery/', ['g-sorcery.cfg'])], license = 'GPL', ) |