diff options
Diffstat (limited to 'bin/make_localpi.py')
| -rwxr-xr-x | bin/make_localpi.py | 244 |
1 files changed, 244 insertions, 0 deletions
diff --git a/bin/make_localpi.py b/bin/make_localpi.py new file mode 100755 index 0000000..7661bb6 --- /dev/null +++ b/bin/make_localpi.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Utility script to create a pypi-like directory structure (localpi) +from a number of Python packages in a directory of the local filesystem. + + DIRECTORY STRUCTURE (before): + +-- downloads/ + +-- alice-1.0.zip + +-- alice-1.0.tar.gz + +-- bob-1.3.0.tar.gz + +-- bob-1.4.2.tar.gz + +-- charly-1.0.tar.bz2 + + DIRECTORY STRUCTURE (afterwards): + +-- downloads/ + +-- simple/ + | +-- alice/index.html --> ../../alice-*.* + | +-- bob/index.html --> ../../bob-*.* + | +-- charly/index.html --> ../../charly-*.* + | +-- index.html --> alice/, bob/, ... + +-- alice-1.0.zip + +-- alice-1.0.tar.gz + +-- bob-1.3.0.tar.gz + +-- bob-1.4.2.tar.gz + +-- charly-1.0.tar.bz2 + +USAGE EXAMPLE: + + mkdir -p /tmp/downloads + pip install --download=/tmp/downloads argparse Jinja2 + make_localpi.py /tmp/downloads + pip install --index-url=file:///tmp/downloads/simple argparse Jinja2 + +ALTERNATIVE: + + pip install --download=/tmp/downloads argparse Jinja2 + pip install --find-links=/tmp/downloads --no-index argparse Jinja2 +""" + +from __future__ import with_statement, print_function +from fnmatch import fnmatch +import os.path +import shutil +import sys + + +__author__ = "Jens Engel" +__version__ = "0.2" +__license__ = "BSD" +__copyright__ = "(c) 2013 by Jens Engel" + + +class Package(object): + """ + Package entity that keeps track of: + * one or more versions of this package + * one or more archive types + """ + PATTERNS = [ + "*.egg", "*.exe", "*.whl", "*.zip", "*.tar.gz", "*.tar.bz2", "*.7z" + ] + + def __init__(self, filename, name=None): + if not name and filename: + name = self.get_pkgname(filename) + self.name = name + self.files = [] + if filename: + self.files.append(filename) + + @property + def versions(self): + versions_info = [ self.get_pkgversion(p) for p in self.files ] + return versions_info + + @classmethod + def get_pkgversion(cls, filename): + parts = os.path.basename(filename).rsplit("-", 1) + version = "" + if len(parts) >= 2: + version = parts[1] + for pattern in cls.PATTERNS: + assert pattern.startswith("*") + suffix = pattern[1:] + if version.endswith(suffix): + version = version[:-len(suffix)] + break + return version + + @staticmethod + def get_pkgname(filename): + name = os.path.basename(filename).rsplit("-", 1)[0] + if name.startswith("http%3A") or name.startswith("https%3A"): + # -- PIP DOWNLOAD-CACHE PACKAGE FILE NAME SCHEMA: + pos = name.rfind("%2F") + name = name[pos+3:] + return name + + @staticmethod + def splitext(filename): + fname = os.path.splitext(filename)[0] + if fname.endswith(".tar"): + fname = os.path.splitext(fname)[0] + return fname + + @classmethod + def isa(cls, filename): + basename = os.path.basename(filename) + if basename.startswith("."): + return False + for pattern in cls.PATTERNS: + if fnmatch(filename, pattern): + return True + return False + + +def make_index_for(package, index_dir, verbose=True): + """ + Create an 'index.html' for one package. + + :param package: Package object to use. + :param index_dir: Where 'index.html' should be created. + """ + index_template = """\ +<html> +<head><title>{title}</title></head> +<body> +<h1>{title}</h1> +<ul> +{packages} +</ul> +</body> +</html> +""" + item_template = '<li><a href="{1}">{0}</a></li>' + index_filename = os.path.join(index_dir, "index.html") + if not os.path.isdir(index_dir): + os.makedirs(index_dir) + + parts = [] + for pkg_filename in package.files: + pkg_name = os.path.basename(pkg_filename) + if pkg_name == "index.html": + # -- ROOT-INDEX: + pkg_name = os.path.basename(os.path.dirname(pkg_filename)) + else: + pkg_name = package.splitext(pkg_name) + pkg_relpath_to = os.path.relpath(pkg_filename, index_dir) + parts.append(item_template.format(pkg_name, pkg_relpath_to)) + + if not parts: + print("OOPS: Package %s has no files" % package.name) + return + + if verbose: + root_index = not Package.isa(package.files[0]) + if root_index: + info = "with %d package(s)" % len(package.files) + else: + package_versions = sorted(set(package.versions)) + info = ", ".join(reversed(package_versions)) + message = "%-30s %s" % (package.name, info) + print(message) + + with open(index_filename, "w") as f: + packages = "\n".join(parts) + text = index_template.format(title=package.name, packages=packages) + f.write(text.strip()) + f.close() + + +def make_package_index(download_dir): + """ + Create a pypi server like file structure below download directory. + + :param download_dir: Download directory with packages. + + EXAMPLE BEFORE: + +-- downloads/ + +-- alice-1.0.zip + +-- alice-1.0.tar.gz + +-- bob-1.3.0.tar.gz + +-- bob-1.4.2.tar.gz + +-- charly-1.0.tar.bz2 + + EXAMPLE AFTERWARDS: + +-- downloads/ + +-- simple/ + | +-- alice/index.html --> ../../alice-*.* + | +-- bob/index.html --> ../../bob-*.* + | +-- charly/index.html --> ../../charly-*.* + | +-- index.html --> alice/index.html, bob/index.html, ... + +-- alice-1.0.zip + +-- alice-1.0.tar.gz + +-- bob-1.3.0.tar.gz + +-- bob-1.4.2.tar.gz + +-- charly-1.0.tar.bz2 + """ + if not os.path.isdir(download_dir): + raise ValueError("No such directory: %r" % download_dir) + + pkg_rootdir = os.path.join(download_dir, "simple") + if os.path.isdir(pkg_rootdir): + shutil.rmtree(pkg_rootdir, ignore_errors=True) + os.mkdir(pkg_rootdir) + + # -- STEP: Collect all packages. + package_map = {} + packages = [] + for filename in sorted(os.listdir(download_dir)): + if not Package.isa(filename): + continue + pkg_filepath = os.path.join(download_dir, filename) + package_name = Package.get_pkgname(pkg_filepath) + package = package_map.get(package_name, None) + if not package: + # -- NEW PACKAGE DETECTED: Store/register package. + package = Package(pkg_filepath) + package_map[package.name] = package + packages.append(package) + else: + # -- SAME PACKAGE: Collect other variant/version. + package.files.append(pkg_filepath) + + # -- STEP: Make local PYTHON PACKAGE INDEX. + root_package = Package(None, "Python Package Index") + root_package.files = [ os.path.join(pkg_rootdir, pkg.name, "index.html") + for pkg in packages ] + make_index_for(root_package, pkg_rootdir) + for package in packages: + index_dir = os.path.join(pkg_rootdir, package.name) + make_index_for(package, index_dir) + + +# ----------------------------------------------------------------------------- +# MAIN: +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + if (len(sys.argv) != 2) or "-h" in sys.argv[1:] or "--help" in sys.argv[1:]: + print("USAGE: %s DOWNLOAD_DIR" % os.path.basename(sys.argv[0])) + print(__doc__) + sys.exit(1) + make_package_index(sys.argv[1]) |