Permalink
Browse files

Adding some more documentation and the pip2pi command!

--HG--
rename : pip2pi/__init__.py => libpip2pi/__init__.py
rename : pip2pi/commands.py => libpip2pi/commands.py
  • Loading branch information...
wolever committed Dec 22, 2011
1 parent d646b58 commit 02d7b20a5a90455205517b98d96f5fd8dc91a5a4
Showing with 111 additions and 70 deletions.
  1. +0 −1 .hgignore
  2. +23 −3 README.rst
  3. +1 −1 dir2pi
  4. 0 {pip2pi → libpip2pi}/__init__.py
  5. +82 −16 {pip2pi → libpip2pi}/commands.py
  6. +0 −20 pip2dijkstra
  7. +3 −0 pip2pi
  8. +1 −1 pip2tgz
  9. +0 −27 pkg2dijkstra
  10. +1 −1 setup.py
View
@@ -8,7 +8,6 @@ syntax: glob
.DS_Store
bin-debug/*
bin-release/*
-bin/*
tags
*.beam
*.dump
View
@@ -6,12 +6,16 @@ tarballs can take a long time. ``pip2pi`` helps to alleviate these problems by
making it blindingly simple to maintain a PyPI-compatible repository of packages
your software depends on.
+
Status
------
These tools were developed to be used internally, and they appear to work for
-me. I'm going to be polishing them up a bit, turning them into a proper Python
-package, etc... But they should be functional for now.
+me. A quick glance at the code will make it obvious that they are far from
+robust (ex, they probably won't work on Windows and they make a few calls to
+shell commands that could be implemented in Python)... But they should work,
+and they shouldn't eat your data or steal private keys or anything.
+
Requirements
------------
@@ -47,12 +51,13 @@ can be specified in any format that ``pip`` recognizes::
$ cat requirements.txt
foo==1.2
http://example.com/baz-0.3.tar.gz
- $ pip2tgz packages/ -r requirements.txt
+ $ pip2tgz packages/ -r requirements.txt bam-2.3/
...
$ ls packages/
foo-1.2.tar.gz
bar-0.8.tar.gz
baz-0.3.tar.gz
+ bam-2.3.tar.gz
Building a Package Index
@@ -80,6 +85,20 @@ A directory full of ``.tar.gz`` files can be turned into PyPI-compatible
packages/simple/foo/foo-1.2.tar.gz
+But that's a lot of work...
+---------------------------
+
+If running two commands seems like too much work... Take heart! The ``pip2pi``
+command will run both of them for you, **and** it will even use ``rsync`` copy
+the new packages and index to a remote host!
+
+::
+ $ pip2pi example.com:/var/www/packages/ foo==1.2
+ ...
+ $ curl -I http://example.com/packages/simple/foo/foo-1.2.tar.gz | head -n1
+ HTTP/1.1 200 OK
+
+
Using Your New Package Index
----------------------------
@@ -94,6 +113,7 @@ Or, once it has been mirrored, prefix you ``requirements.txt`` with
--use-index=http://example.com/packages/simple/
foo==1.2
+
Keywords
========
View
2 dir2pi
@@ -1,3 +1,3 @@
#!/usr/bin/env python
-from pip2pi.commands import dir2pi
+from libpip2pi.commands import dir2pi
dir2pi()
File renamed without changes.
@@ -1,11 +1,13 @@
import os
import sys
import shutil
+import atexit
+import tempfile
import textwrap
from subprocess import check_call
def dedent(text):
- return textwrap.dedent(text)
+ return textwrap.dedent(text.lstrip("\n"))
def file_to_package(file):
""" Returns the package name for a given file.
@@ -22,10 +24,10 @@ def file_to_package(file):
raise ValueError(msg)
return (split[0], split[1].replace("_", "-"))
-def dir2pi():
- if len(sys.argv) != 2:
+def dir2pi(argv=sys.argv):
+ if len(argv) != 2:
print dedent("""
- usage: %s PACKAGE_DIR
+ usage: dir2pi PACKAGE_DIR
Creates the directory PACKAGE_DIR/simple/ and populates it with the
directory structure required to use with pip's --index-url.
@@ -36,9 +38,21 @@ def dir2pi():
This makes the most sense if PACKAGE_DIR is somewhere inside a
webserver's inside htdocs directory.
- """ %(sys.argv[0], ) )
+
+ For example:
+
+ $ ls packages/
+ foo-1.2.tar.gz
+ $ dir2pi packags/

This comment has been minimized.

Show comment Hide comment
@bernardofire

bernardofire Feb 24, 2012

Typo: packags/

This comment has been minimized.

Show comment Hide comment
@wolever

wolever Feb 24, 2012

Owner

Thanks! Fixed.

@wolever

wolever Feb 24, 2012

Owner

Thanks! Fixed.

+ $ find packages/
+ packages/
+ packages/foo-1.2.tar.gz
+ packages/simple/
+ packages/simple/foo/
+ packages/simple/foo/foo-1.2.tar.gz
+ """)
sys.exit(1)
- pkgdir = sys.argv[1]
+ pkgdir = argv[1]
if not os.path.isdir(pkgdir):
raise ValueError("no such directory: %r" %(pkgdir, ))
pkgdirpath = lambda *x: os.path.join(pkgdir, *x)
@@ -56,37 +70,44 @@ def dir2pi():
if not os.path.exists(pkg_dir):
os.mkdir(pkg_dir)
pkg_new_basename = "-".join([pkg_name, pkg_rest])
- os.link(pkg_filepath, os.path.join(pkg_dir, pkg_new_basename))
+ symlink_target = os.path.join(pkg_dir, pkg_new_basename)
+ symlink_source = os.path.join("../../", pkg_new_basename)
+ os.symlink(symlink_source, symlink_target)
-def pip2tgz():
- if len(sys.argv) < 3:
+def pip2tgz(argv=sys.argv):
+ if len(argv) < 3:
print dedent("""
- usage: %s OUTPUT_DIRECTORY PACKAGE_NAMES
+ usage: pip2tgz OUTPUT_DIRECTORY PACKAGE_NAME ...
Where PACKAGE_NAMES are any names accepted by pip (ex, `foo`,
`foo==1.2`, `-r requirements.txt`).
- %s will download all packages required to install PACKAGE_NAMES and
+ pip2tgz will download all packages required to install PACKAGE_NAMES and
save them to sanely-named tarballs in OUTPUT_DIRECTORY.
- """ %(sys.argv[0], sys.argv[0]))
+
+ For example:
+
+ $ pip2tgz /var/www/packages/ -r requirements.txt foo==1.2 baz/
+ """)
sys.exit(1)
- outdir = os.path.abspath(sys.argv[1])
+ outdir = os.path.abspath(argv[1])
if not os.path.exists(outdir):
- os.path.mkdir(outdir)
+ os.mkdir(outdir)
tempdir = os.path.join(outdir, "_pip2tgz_temp")
if os.path.exists(tempdir):
shutil.rmtree(tempdir)
os.mkdir(tempdir)
bundle_zip = os.path.join(tempdir, "bundle.zip")
- check_call(["pip", "bundle", "-b", tempdir, bundle_zip] + sys.argv[2:])
+ check_call(["pip", "bundle", "-b", tempdir, bundle_zip] + argv[2:])
os.chdir(tempdir)
check_call(["unzip", "bundle.zip", "pip-manifest.txt"])
+ num_pakages = 0
for line in open("pip-manifest.txt"):
line = line.strip()
if not line or line.startswith("#"):
@@ -102,7 +123,52 @@ def pip2tgz():
new_input_dir = "%s-%s" %(pkg, version)
os.rename(old_input_dir, new_input_dir)
output_name = os.path.join("..", new_input_dir + ".tar.gz")
- check_call(["tar", "-czvf", output_name, new_input_dir])
+ check_call(["tar", "-czf", output_name, new_input_dir])
+ num_pakages += 1
os.chdir(outdir)
shutil.rmtree(tempdir)
+ print "%s .tar.gz saved to %r" %(num_pakages, argv[1])
+
+def pip2pi(argv=sys.argv):
+ if len(argv) < 3:
+ print dedent("""
+ usage: pip2pi TARGET PACKAGE_NAME ...
+
+ Combines pip2tgz and dir2pi, adding PACKAGE_NAME to package index
+ TARGET.
+
+ If TARGET contains ':' it will be treated as a remote path. The
+ package index will be built locally then rsync will be used to copy
+ it to the remote host.
+
+ For example, to create a remote index:
+
+ $ pip2pi example.com:/var/www/packages/ -r requirements.txt
+
+ Or to create a local index:
+
+ $ pip2pi ~/Sites/packages/ foo==1.2
+ """)
+ sys.exit(1)
+
+ target = argv[1]
+ pip_packages = argv[2:]
+ if ":" in target:
+ is_remote = True
+ working_dir = tempfile.mkdtemp(prefix="pip2pi-working-dir")
+ atexit.register(lambda: shutil.rmtree(working_dir))
+ else:
+ is_remote = False
+ working_dir = target
+
+ pip2tgz([argv[0], working_dir] + pip_packages)
+ dir2pi([argv[0], working_dir])
+
+ if is_remote:
+ print "copying temporary index at %r to %r..." %(working_dir, target)
+ check_call([
+ "rsync",
+ "--recursive", "--progress", "--links",
+ working_dir + "/", target + "/",
+ ])
View
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-IFS="`printf "\n\t"`"
-set -eu
-
-if [[ "$#" -lt 1 ]]; then
- echo "usage: $0 PACKAGE_NAME ..."
- exit 1
-fi
-
-pip2tgz="$(dirname "$0")/pip2tgz"
-pkg2dijkstra="$(dirname "$0")/pkg2dijkstra"
-
-tempdir="/tmp/_pip2dijkstra_temp/"
-mkdir "$tempdir" 2>&1 > /dev/null
-
-$pip2tgz "$tempdir" "$@"
-files=($tempdir/*)
-$pkg2dijkstra "${files[@]}"
-rm -r "$tempdir"
View
3 pip2pi
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from libpip2pi.commands import pip2pi
+pip2pi()
View
@@ -1,3 +1,3 @@
#!/usr/bin/env python
-from pip2pi.commands import pip2tgz
+from libpip2pi.commands import pip2tgz
pip2tgz()
View
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-IFS="`printf "\n\t"`"
-set -eu
-
-if [[ -z "${1-}" ]]; then
- echo "usage: $0 (foo-1.2.3.tar.gz|foo/) ..."
- exit 1
-fi
-
-targets=("$@")
-for (( i = 0; i < $#; i += 1)); do
- target="${targets[$i]}"
- if [[ -d "$target" ]]; then
- echo "looks like a source directory. Doing some magic to make that work..."
- pushd "$target" > /dev/null
- rm -r dist/ 2>&1 > /dev/null || true
- python setup.py sdist
- target="$(echo "$PWD/dist/"*)"
- popd > /dev/null
- targets[$i]="$target"
- fi
-done
-
-scp "${targets[@]}" luminaut@dijkstra.local:/var/www/pypi/
-
-ssh luminaut@dijkstra.local "cd /var/www/; ./dir2pypi pypi"
View
@@ -15,5 +15,5 @@
author_email="david@wolever.net",
description="pip2pi builds a PyPI-compatible package repository from pip requirements",
packages=find_packages(),
- scripts=["pip2tgz", "dir2pi"],
+ scripts=["pip2pi", "pip2tgz", "dir2pi"],
)

0 comments on commit 02d7b20

Please sign in to comment.