Permalink
Browse files

Merge branch 'url-spec' into devel

  • Loading branch information...
2 parents e2297e2 + 34e5c9e commit 5635ba96ed9d993094789c3a3dc22c1f5fd7d520 @dvarrazzo committed Aug 30, 2012
View
4 CHANGES
@@ -6,7 +6,9 @@ PGXN Client changes log
pgxnclient 1.2
==============
-- ``gmake`` if used in favour of ``make`` for platforms where the two are
+- Packages can be downloaded, installed, loaded specifying an URL
+ (ticket #15).
+- Use ``gmake`` in favour of ``make`` for platforms where the two are
distinct, such as BSD (ticket #14).
- Added ``--make`` option to select the make executable (ticket #16).
View
38 docs/usage.rst
@@ -65,6 +65,9 @@ directory containing a distribution: in this case the specification should
contain at least a path separator to disambiguate it from a distribution name,
for instance ``pgxn install ./foo.zip``.
+A few commands also allow specifying a package with an URL. Currently the
+schemas ``http://`` and ``https://`` are supported.
+
.. _install:
@@ -85,7 +88,8 @@ Usage:
The program takes a `package specification`_ identifying the distribution to
work with. The download phase is skipped if the distribution specification
-refers to a local directory or package.
+refers to a local directory or package. The package may be specified with an
+URL.
Note that the built extension is not loaded in any database: use the command
`load`_ for this purpose.
@@ -144,8 +148,9 @@ Usage:
*SPEC*
The command takes a `package specification`_ identifying the distribution to
-work with, which can also be a local file or directory. The distribution is
-unpacked if required and the ``installcheck`` make target is run.
+work with, which can also be a local file or directory or an URL. The
+distribution is unpacked if required and the ``installcheck`` make target is
+run.
.. note::
The command doesn't run ``make all`` before ``installcheck``: if any file
@@ -224,11 +229,11 @@ Usage:
*SPEC* [*EXT* [*EXT* ...]]
The distribution is specified according to the `package specification`_ and
-can refer to a local directory or file. No consistency check is performed
-between the packages specified in the ``install`` and ``load`` command: the
-specifications should refer to compatible packages. The specified distribution
-is only used to read the metadata: only installed files are actually used to
-issue database commands.
+can refer to a local directory or file or to an URL. No consistency check is
+performed between the packages specified in the ``install`` and ``load``
+command: the specifications should refer to compatible packages. The specified
+distribution is only used to read the metadata: only installed files are
+actually used to issue database commands.
The database to install into can be specified using options
``-d``/``--dbname``, ``-h``/``--host``, ``-p``/``--port``,
@@ -334,11 +339,12 @@ Usage:
[--target *PATH*]
*SPEC*
-The distribution is specified according to the `package specification`_. The
-file is saved in the current directory with name usually
-:samp:`{distribution}-{version}.zip`. If a file with the same name exists, a
-suffix ``-1``, ``-2`` etc. is added to the name, before the extension. A
-different directory or name can be specified using the ``--target`` option.
+The distribution is specified according to the `package specification`_ and
+can be represented by an URL. The file is saved in the current directory with
+name usually :samp:`{distribution}-{version}.zip`. If a file with the same
+name exists, a suffix ``-1``, ``-2`` etc. is added to the name, before the
+extension. A different directory or name can be specified using the
+``--target`` option.
.. _pgxn-search:
@@ -419,9 +425,9 @@ Usage:
[--details | --meta | --readme | --versions]
*SPEC*
-The distribution is specified according to the `package specification`_.
-The command output is a list of values obtained by the distribution's
-``META.json`` file, for example:
+The distribution is specified according to the `package specification`_. It
+cannot be a local dir or file nor an URL. The command output is a list of
+values obtained by the distribution's ``META.json`` file, for example:
.. code-block:: console
View
6 pgxnclient/api.py
@@ -10,9 +10,9 @@
from urllib import urlencode
+from pgxnclient import network
from pgxnclient.utils import load_json
from pgxnclient.errors import NetworkError, NotFound, ResourceNotFound
-from pgxnclient.network import get_file
from pgxnclient.utils.uri import expand_template
@@ -78,7 +78,7 @@ def user(self, username):
def call(self, meth, args=None, query=None):
url = self.get_url(meth, args, query)
- return get_file(url)
+ return network.get_file(url)
def get_url(self, meth, args=None, query=None):
tmpl = self.get_template(meth)
@@ -98,7 +98,7 @@ def get_index(self):
if self._api_index is None:
url = self.mirror.rstrip('/') + '/index.json'
try:
- with get_file(url) as f:
+ with network.get_file(url) as f:
self._api_index = load_json(f)
except ResourceNotFound:
raise NetworkError("API index not found at '%s'" % url)
View
62 pgxnclient/commands/__init__.py
@@ -20,10 +20,12 @@
from pgxnclient.utils import load_json, argparse, find_executable
from pgxnclient import __version__
+from pgxnclient import network
from pgxnclient import Spec, SemVer
from pgxnclient.api import Api
from pgxnclient.i18n import _, gettext
from pgxnclient.errors import NotFound, PgxnClientException, ProcessError, ResourceNotFound, UserAbort
+from pgxnclient.utils.temp import temp_dir
logger = logging.getLogger('pgxnclient.commands')
@@ -265,7 +267,7 @@ def customize_parser(self, parser, subparsers,
return subp
- def get_spec(self, _can_be_local=False):
+ def get_spec(self, _can_be_local=False, _can_be_url=False):
"""
Return the package specification requested.
@@ -283,6 +285,10 @@ def get_spec(self, _can_be_local=False):
raise PgxnClientException(
_("you cannot use a local resource with this command"))
+ if not _can_be_url and spec.is_url():
+ raise PgxnClientException(
+ _("you cannot use an url with this command"))
+
return spec
def get_best_version(self, data, spec, quiet=False):
@@ -359,7 +365,7 @@ def get_meta(self, spec):
Return the object obtained parsing the JSON.
"""
- if not spec.is_local():
+ if spec.is_name():
# Get the metadata from the API
try:
data = self.api.dist(spec.name)
@@ -387,6 +393,14 @@ def get_meta(self, spec):
# Get the metadata from a zip file
return get_meta_from_zip(spec.filename)
+ elif spec.is_url():
+ with network.get_file(spec.url) as fin:
+ with temp_dir() as dir:
+ fn = network.download(fin, dir)
+ return get_meta_from_zip(fn)
+
+ else:
+ assert False
class WithSpecLocal(WithSpec):
"""
@@ -405,35 +419,29 @@ def customize_parser(self, parser, subparsers, epilog=None, **kwargs):
return subp
- def get_spec(self):
- return super(WithSpecLocal, self).get_spec(_can_be_local=True)
-
+ def get_spec(self, **kwargs):
+ kwargs['_can_be_local'] = True
+ return super(WithSpecLocal, self).get_spec(**kwargs)
-import shutil
-import tempfile
-from pgxnclient.utils.zip import unpack
-
-class WithUnpacking(object):
+class WithSpecUrl(WithSpec):
"""
- Mixin to implement commands that may deal with zip files.
+ Mixin to implement commands that can also refer to a URL.
"""
- def call_with_temp_dir(self, f, *args, **kwargs):
- """
- Call a function in the context of a temporary directory.
- Create the temp directory and pass its name as first argument to *f*.
- Other arguments and keywords are passed to *f* too. Upon exit delete
- the directory.
- """
- dir = tempfile.mkdtemp()
- try:
- return f(dir, *args, **kwargs)
- finally:
- shutil.rmtree(dir)
+ @classmethod
+ def customize_parser(self, parser, subparsers, epilog=None, **kwargs):
+ epilog = _("""
+SPEC may also be an url specifying a protocol such as 'http://' or 'https://'.
+""") + (epilog or "")
+
+ subp = super(WithSpecUrl, self).customize_parser(
+ parser, subparsers, epilog=epilog, **kwargs)
+
+ return subp
- def unpack(self, zipname, destdir):
- """Unpack the zip file *zipname* into *destdir*."""
- return unpack(zipname, destdir)
+ def get_spec(self, **kwargs):
+ kwargs['_can_be_url'] = True
+ return super(WithSpecUrl, self).get_spec(**kwargs)
class WithPgConfig(object):
@@ -489,7 +497,7 @@ def get_pg_config(self):
import shlex
-class WithMake(WithPgConfig, WithUnpacking):
+class WithMake(WithPgConfig):
"""
Mixin to implement commands that should invoke :program:`make`.
"""
View
48 pgxnclient/commands/install.py
@@ -17,18 +17,20 @@
from subprocess import PIPE
from pgxnclient import SemVer
+from pgxnclient import network
from pgxnclient.i18n import _, N_
from pgxnclient.utils import sha1, b
from pgxnclient.errors import BadChecksum, PgxnClientException, InsufficientPrivileges
-from pgxnclient.network import download
from pgxnclient.commands import Command, WithDatabase, WithMake, WithPgConfig
-from pgxnclient.commands import WithSpec, WithSpecLocal, WithSudo
+from pgxnclient.commands import WithSpecUrl, WithSpecLocal, WithSudo
+from pgxnclient.utils.zip import unpack
+from pgxnclient.utils.temp import temp_dir
from pgxnclient.utils.strings import Identifier
logger = logging.getLogger('pgxnclient.commands')
-class Download(WithSpec, Command):
+class Download(WithSpecUrl, Command):
name = 'download'
description = N_("download a distribution from the network")
@@ -43,6 +45,11 @@ def customize_parser(self, parser, subparsers, **kwargs):
def run(self):
spec = self.get_spec()
+ assert not spec.is_local()
+
+ if spec.is_url():
+ return self._run_url(spec)
+
data = self.get_meta(spec)
try:
@@ -52,11 +59,17 @@ def run(self):
"sha1 missing from the distribution meta")
with self.api.download(data['name'], SemVer(data['version'])) as fin:
- fn = self._get_local_file_name(fin.url)
- fn = download(fin, fn, rename=True)
+ fn = network.download(fin, self.opts.target)
+
self.verify_checksum(fn, chk)
return fn
+ def _run_url(self, spec):
+ with network.get_file(spec.url) as fin:
+ fn = network.download(fin, self.opts.target)
+
+ return fn
+
def verify_checksum(self, fn, chk):
"""Verify that a downloaded file has the expected sha1."""
sha = sha1()
@@ -77,34 +90,27 @@ def verify_checksum(self, fn, chk):
fn, sha, chk)
raise BadChecksum(_("bad sha1 in downloaded file"))
- def _get_local_file_name(self, url):
- from urlparse import urlsplit
- if os.path.isdir(self.opts.target):
- basename = urlsplit(url)[2].rsplit('/', 1)[-1]
- fn = os.path.join(self.opts.target, basename)
- else:
- fn = self.opts.target
-
- return os.path.abspath(fn)
-
-class InstallUninstall(WithMake, WithSpecLocal, Command):
+class InstallUninstall(WithMake, WithSpecUrl, WithSpecLocal, Command):
"""
Base class to implement the ``install`` and ``uninstall`` commands.
"""
def run(self):
- return self.call_with_temp_dir(self._run)
+ with temp_dir() as dir:
+ return self._run(dir)
def _run(self, dir):
spec = self.get_spec()
if spec.is_dir():
pdir = os.path.abspath(spec.dirname)
elif spec.is_file():
- pdir = self.unpack(spec.filename, dir)
- else: # download
+ pdir = unpack(spec.filename, dir)
+ elif not spec.is_local():
self.opts.target = dir
fn = Download(self.opts).run()
- pdir = self.unpack(fn, dir)
+ pdir = unpack(fn, dir)
+ else:
+ assert False
self.maybe_run_configure(pdir)
@@ -220,7 +226,7 @@ def _inun(self, pdir):
raise
-class LoadUnload(WithPgConfig, WithDatabase, WithSpecLocal, Command):
+class LoadUnload(WithPgConfig, WithDatabase, WithSpecUrl, WithSpecLocal, Command):
"""
Base class to implement the ``load`` and ``unload`` commands.
"""
View
22 pgxnclient/network.py
@@ -8,6 +8,7 @@
import os
import urllib2
+from urlparse import urlsplit
from itertools import count
from contextlib import closing
@@ -39,16 +40,33 @@ def get_file(url):
except urllib2.URLError, e:
raise NetworkError(_("network error: %s") % e.reason)
+def get_local_file_name(target, url):
+ """Return a good name for a local file.
+
+ If *target* is a dir, make a name out of the url. Otherwise return target
+ itself. Always return an absolute path.
+ """
+ if os.path.isdir(target):
+ basename = urlsplit(url)[2].rsplit('/', 1)[-1]
+ fn = os.path.join(target, basename)
+ else:
+ fn = target
+
+ return os.path.abspath(fn)
+
def download(f, fn, rename=True):
"""Download a file locally.
- :param f: open file read
- :param fn: name of the file to write
+ :param f: open file to read
+ :param fn: name of the file to write. If a dir, save into it.
:param rename: if true and a file *fn* exist, rename the downloaded file
adding a prefix ``-1``, ``-2``... before the extension.
Return the name of the file saved.
"""
+ if os.path.isdir(fn):
+ fn = get_local_file_name(fn, f.url)
+
if rename:
if os.path.exists(fn):
base, ext = os.path.splitext(fn)
View
17 pgxnclient/spec.py
@@ -33,26 +33,33 @@ class Spec(object):
'stable': STABLE, }
def __init__(self, name=None, op=None, ver=None,
- dirname=None, filename=None):
+ dirname=None, filename=None, url=None):
self.name = name and name.lower()
self.op = op
self.ver = ver
- # point to local files
+ # point to local files or specific resources
self.dirname = dirname
self.filename = filename
+ self.url = url
+
+ def is_name(self):
+ return self.name is not None
def is_dir(self):
return self.dirname is not None
def is_file(self):
return self.filename is not None
+ def is_url(self):
+ return self.url is not None
+
def is_local(self):
return self.is_dir() or self.is_file()
def __str__(self):
- name = self.name or self.filename or self.dirname or "???"
+ name = self.name or self.filename or self.dirname or self.url or "???"
if self.op is None:
return name
else:
@@ -64,6 +71,10 @@ def parse(self, spec):
Raise BadSpecError if couldn't parse.
"""
+ # TODO: handle file:// too
+ if spec.startswith('http://') or spec.startswith('https://'):
+ return Spec(url=spec)
+
if os.sep in spec:
# This is a local thing, let's see what
if os.path.isdir(spec):
View
80 pgxnclient/tests/test_commands.py
@@ -44,7 +44,7 @@ def f(what):
class InfoTestCase(unittest.TestCase):
def _get_output(self, cmdline):
@patch('sys.stdout')
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def do(mock, stdout):
mock.side_effect = fake_get_file
from pgxnclient.cli import main
@@ -120,7 +120,7 @@ def test_popen_raises(self):
class DownloadTestCase(unittest.TestCase):
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_download_latest(self, mock):
mock.side_effect = fake_get_file
@@ -134,7 +134,7 @@ def test_download_latest(self, mock):
finally:
ifunlink(fn)
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_download_testing(self, mock):
mock.side_effect = fake_get_file
@@ -148,7 +148,21 @@ def test_download_testing(self, mock):
finally:
ifunlink(fn)
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
+ def test_download_url(self, mock):
+ mock.side_effect = fake_get_file
+
+ fn = 'foobar-0.43.2b1.zip'
+ self.assert_(not os.path.exists(fn))
+
+ from pgxnclient.cli import main
+ try:
+ main(['download', 'http://api.pgxn.org/dist/foobar/0.43.2b1/foobar-0.43.2b1.zip'])
+ self.assert_(os.path.exists(fn))
+ finally:
+ ifunlink(fn)
+
+ @patch('pgxnclient.network.get_file')
def test_download_ext(self, mock):
mock.side_effect = fake_get_file
@@ -162,7 +176,7 @@ def test_download_ext(self, mock):
finally:
ifunlink(fn)
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_download_rename(self, mock):
mock.side_effect = fake_get_file
@@ -195,7 +209,7 @@ def test_download_rename(self, mock):
ifunlink(fn1)
ifunlink(fn2)
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_download_bad_sha1(self, mock):
def fakefake(url):
return fake_get_file(url, urlmap = {
@@ -218,7 +232,7 @@ def fakefake(url):
finally:
ifunlink(fn)
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_download_case_insensitive(self, mock):
mock.side_effect = fake_get_file
@@ -325,7 +339,7 @@ def assertCallArgs(self, pattern, args):
class InstallTestCase(unittest.TestCase, Assertions):
def setUp(self):
- self._p1 = patch('pgxnclient.api.get_file')
+ self._p1 = patch('pgxnclient.network.get_file')
self.mock_get = self._p1.start()
self.mock_get.side_effect = fake_get_file
@@ -366,6 +380,17 @@ def test_install_local(self):
self.assertCallArgs([self.make], self.mock_popen.call_args_list[0][0][0][:1])
self.assertCallArgs([self.make], self.mock_popen.call_args_list[1][0][0][:1])
+ def test_install_url(self):
+ self.mock_pgconfig.side_effect = fake_pg_config(
+ libdir=os.environ['HOME'], bindir='/')
+
+ from pgxnclient.cli import main
+ main(['install', 'http://api.pgxn.org/dist/foobar/0.42.1/foobar-0.42.1.zip'])
+
+ self.assertEquals(self.mock_popen.call_count, 2)
+ self.assertCallArgs([self.make], self.mock_popen.call_args_list[0][0][0][:1])
+ self.assertCallArgs([self.make], self.mock_popen.call_args_list[1][0][0][:1])
+
def test_install_fails(self):
self.mock_popen.return_value.returncode = 1
self.mock_pgconfig.side_effect = fake_pg_config(
@@ -409,7 +434,7 @@ def test_install_sudo(self):
self.assertCallArgs(['gksudo', '-d', 'hello world', self.make],
self.mock_popen.call_args_list[1][0][0][:4])
- @patch('pgxnclient.commands.unpack')
+ @patch('pgxnclient.commands.install.unpack')
def test_install_local_zip(self, mock_unpack):
from pgxnclient.utils.zip import unpack
mock_unpack.side_effect = unpack
@@ -452,7 +477,7 @@ def test_install_local_dir(self):
class CheckTestCase(unittest.TestCase, Assertions):
def setUp(self):
- self._p1 = patch('pgxnclient.api.get_file')
+ self._p1 = patch('pgxnclient.network.get_file')
self.mock_get = self._p1.start()
self.mock_get.side_effect = fake_get_file
@@ -477,6 +502,13 @@ def test_check_latest(self):
self.assertEquals(self.mock_popen.call_count, 1)
self.assertCallArgs([self.make], self.mock_popen.call_args_list[0][0][0][:1])
+ def test_check_url(self):
+ from pgxnclient.cli import main
+ main(['check', 'http://api.pgxn.org/dist/foobar/0.42.1/foobar-0.42.1.zip'])
+
+ self.assertEquals(self.mock_popen.call_count, 1)
+ self.assertCallArgs([self.make], self.mock_popen.call_args_list[0][0][0][:1])
+
def test_check_fails(self):
self.mock_popen.return_value.returncode = 1
@@ -558,7 +590,7 @@ def test_parse_version(self):
'PostgreSQL 9.1alpha5 on i686-pc-linux-gnu, compiled by GCC gcc'
' (Ubuntu/Linaro 4.4.4-14ubuntu5) 4.4.5, 32-bit '))
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_check_psql_options(self, mock_get):
mock_get.side_effect = fake_get_file
@@ -580,8 +612,8 @@ def test_check_psql_options(self, mock_get):
args = self.mock_popen.call_args[0][0]
self.assertEqual('somewhere', args[args.index('--host') + 1])
- @patch('pgxnclient.commands.unpack')
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.commands.install.unpack')
+ @patch('pgxnclient.network.get_file')
def test_load_local_zip(self, mock_get, mock_unpack):
mock_get.side_effect = lambda *args: self.fail('network invoked')
from pgxnclient.utils.zip import unpack
@@ -597,7 +629,7 @@ def test_load_local_zip(self, mock_get, mock_unpack):
self.assertEquals(communicate.call_args[0][0],
'CREATE EXTENSION foobar;')
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_load_local_dir(self, mock_get):
mock_get.side_effect = lambda *args: self.fail('network invoked')
@@ -618,6 +650,24 @@ def test_load_local_dir(self, mock_get):
self.assertEquals(communicate.call_args[0][0],
'CREATE EXTENSION foobar;')
+ @patch('pgxnclient.commands.install.unpack')
+ @patch('pgxnclient.network.get_file')
+ def test_load_url(self, mock_get, mock_unpack):
+ mock_get.side_effect = fake_get_file
+ from pgxnclient.utils.zip import unpack
+ mock_unpack.side_effect = unpack
+
+ from pgxnclient.cli import main
+ main(['load', '--yes',
+ 'http://api.pgxn.org/dist/foobar/0.42.1/foobar-0.42.1.zip'])
+
+ self.assertEquals(mock_unpack.call_count, 0)
+ self.assertEquals(self.mock_popen.call_count, 1)
+ self.assert_('psql' in self.mock_popen.call_args[0][0][0])
+ communicate = self.mock_popen.return_value.communicate
+ self.assertEquals(communicate.call_args[0][0],
+ 'CREATE EXTENSION foobar;')
+
def test_load_extensions_order(self):
tdir = tempfile.mkdtemp()
try:
@@ -757,7 +807,7 @@ def test_unload_missing(self):
class SearchTestCase(unittest.TestCase):
@patch('sys.stdout')
- @patch('pgxnclient.api.get_file')
+ @patch('pgxnclient.network.get_file')
def test_search_quoting(self, mock_get, stdout):
mock_get.side_effect = fake_get_file
from pgxnclient.cli import main
View
19 pgxnclient/utils/temp.py
@@ -0,0 +1,19 @@
+"""
+pgxnclient -- temp files utilities
+"""
+
+# Copyright (C) 2011-2012 Daniele Varrazzo
+
+# This file is part of the PGXN client
+
+import shutil
+import tempfile
+import contextlib
+
+@contextlib.contextmanager
+def temp_dir():
+ """Context manager to create a temp dir and delete after usage."""
+ dir = tempfile.mkdtemp()
+ yield dir
+ shutil.rmtree(dir)
+

0 comments on commit 5635ba9

Please sign in to comment.