SqlStore: Allow specifying the table name in DB URLs using the URL "f…

…ragment".

The precise syntax for specifying the table is to use a fragment
formatted like `#table=name`, e.g., `sqlite:////tmp/jobs.db#table=tasks`.

Optional argument `table` overrides the fragment if it's not ``None``.

gc3libs/persistence/sql.py

@@ -3,7 +3,7 @@
 """
 SQL-based storage of GC3pie objects.
 """
-# Copyright (C) 2011-2012, 2015 S3IT, Zentrale Informatik, University of Zurich. All rights reserved.
+# Copyright (C) 2011-2018 University of Zurich. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -26,6 +26,7 @@
 from contextlib import closing
 from cStringIO import StringIO
 import os
+from urlparse import parse_qs
 from warnings import warn
 
 import sqlalchemy as sqla
@@ -34,6 +35,7 @@
 # GC3Pie interface
 from gc3libs import Run
 import gc3libs.exceptions
+from gc3libs.url import Url
 import gc3libs.utils
 from gc3libs.utils import same_docstring_as
 
@@ -139,7 +141,7 @@ class SqlStore(Store):
     `FilesystemStore`:class:.
     """
 
-    def __init__(self, url, table_name="store", idfactory=None,
+    def __init__(self, url, table_name=None, idfactory=None,
                  extra_fields=None, create=True, **extra_args):
         """
         Open a connection to the storage database identified by `url`.
@@ -148,13 +150,26 @@ def __init__(self, url, table_name="store", idfactory=None,
         `url.scheme` value.
         """
         super(SqlStore, self).__init__(url)
+        if self.url.fragment:
+            kv = parse_qs(self.url.fragment)
+        else:
+            kv = {}
 
         # init static public args
         if not idfactory:
             self.idfactory = IdFactory(id_class=IntId)
         else:
             self.idfactory = idfactory
-        self.table_name = table_name
+
+        url_table_name = kv.get('table', "store")
+        if table_name is None:
+            self.table_name = url_table_name
+        else:
+            if table_name != url_table_name:
+                gc3libs.log.debug(
+                    "DB table name given in store URL fragment,"
+                    " but overriden by `table` argument to SqlStore()")
+            self.table_name = table_name
 
         # save ctor args for lazy-initialization
         self._init_extra_fields = (extra_fields if extra_fields is not None else {})
@@ -165,6 +180,22 @@ def __init__(self, url, table_name="store", idfactory=None,
         self._real_extra_fields = None
         self._real_tables = None
 
+    @staticmethod
+    def _to_sqlalchemy_url(url):
+        if url.scheme == 'sqlite':
+            # rewrite ``sqlite`` URLs to be RFC compliant, see:
+            # https://github.com/uzh/gc3pie/issues/261
+            db_url = "%s://%s/%s" % (url.scheme, url.netloc, url.path)
+        else:
+            db_url = str(url)
+        # remove fragment identifier, if any
+        try:
+            fragment_loc = db_url.index('#')
+            db_url = db_url[:fragment_loc]
+        except ValueError:
+            pass
+        return db_url
+
     def _delayed_init(self):
         """
         Perform initialization tasks that can interfere with
@@ -174,7 +205,8 @@ def _delayed_init(self):
         <https://github.com/uzh/gc3pie/issues/550>`_ for more details
         and motivation.
         """
-        self._real_engine = sqla.create_engine(str(self.url))
+        self._real_engine = sqla.create_engine(
+            self._to_sqlalchemy_url(self.url))
 
         # create schema
         meta = sqla.MetaData(bind=self._real_engine)
@@ -336,16 +368,14 @@ def make_sqlstore(url, *args, **extra_args):
 
     """
     assert isinstance(url, gc3libs.url.Url)
+    gc3libs.log.debug("Building SQL store from URL %s", url)
     if url.scheme == 'sqlite':
         # create parent directories: avoid "OperationalError: unable to open
         # database file None None"
         dir = gc3libs.utils.dirname(url.path)
         if not os.path.exists(dir):
             os.makedirs(dir)
-        # rewrite ``sqlite`` URLs to be RFC compliant, see:
-        # https://github.com/uzh/gc3pie/issues/261
-        url = "%s://%s/%s" % (url.scheme, url.netloc, url.path)
-    return SqlStore(str(url), *args, **extra_args)
+    return SqlStore(url, *args, **extra_args)
 
 
 # main: run tests

gc3libs/persistence/store.py

@@ -2,7 +2,7 @@
 #
 """
 """
-# Copyright (C) 2011-2015 S3IT, Zentrale Informatik, University of Zurich. All rights reserved.
+# Copyright (C) 2011-2018 University of Zurich. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -46,6 +46,8 @@ class Store(object):
     """
 
     def __init__(self, url=None):
+        if url and not isinstance(url, Url):
+            url = Url(url)
         self.url = url
 
     def list(self, **extra_args):

gc3libs/session.py

@@ -2,7 +2,7 @@
 #
 """
 """
-# Copyright (C) 2012, 2015 S3IT, Zentrale Informatik, University of Zurich. All rights reserved.
+# Copyright (C) 2012-2018 University of Zurich. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -229,9 +229,10 @@ def _load_session(self, **extra_args):
         try:
             store_fname = os.path.join(self.path, self.STORE_URL_FILENAME)
             self.store_url = gc3libs.utils.read_contents(store_fname).strip()
+            gc3libs.log.debug("Loading session from URL %s ...", self.store_url)
         except IOError:
             gc3libs.log.info(
-                "Unable to load session: file %s is missing." % (store_fname))
+                "Unable to load session: file %s is missing.", store_fname)
             raise
         if 'store' in extra_args:
             self.store = extra_args['store']

gc3libs/url.py

@@ -3,7 +3,7 @@
 """
 Utility classes and methods for dealing with URLs.
 """
-# Copyright (C) 2011-2015 S3IT, Zentrale Informatik, University of Zurich. All rights reserved.
+# Copyright (C) 2011-2018 University of Zurich. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -46,6 +46,7 @@ class Url(tuple):
     port        5       Port number as integer (if present)  None
     username    6       User name                            None
     password    7       Password                             None
+    fragment    8       URL fragment (part after ``#``)      empty string
     =========   =====   ===================================  ==================
 
     There are two ways of constructing `Url` objects:
@@ -128,11 +129,13 @@ class Url(tuple):
     __slots__ = ()
 
     _fields = ['scheme', 'netloc', 'path',
-               'hostname', 'port', 'query', 'username', 'password']
+               'hostname', 'port', 'query',
+               'username', 'password', 'fragment']
 
     def __new__(cls, urlstring=None, force_abs=True,
-                scheme='file', netloc='', path='', 
-                hostname=None, port=None, query='', username=None, password=None):
+                scheme='file', netloc='', path='',
+                hostname=None, port=None, query='',
+                username=None, password=None, fragment=''):
         """
         Create a new `Url` object.  See the `Url`:class: documentation
         for invocation syntax.
@@ -143,7 +146,7 @@ def __new__(cls, urlstring=None, force_abs=True,
                 return tuple.__new__(cls, (
                     urlstring.scheme, urlstring.netloc, urlstring.path,
                     urlstring.hostname, urlstring.port, urlstring.query,
-                    urlstring.username, urlstring.password
+                    urlstring.username, urlstring.password, urlstring.fragment
                 ))
             else:
                 # parse `urlstring` and use kwd arguments as default values
@@ -163,6 +166,7 @@ def __new__(cls, urlstring=None, force_abs=True,
                         urldata.query or query,
                         urldata.username or username,
                         urldata.password or password,
+                        urldata.fragment or fragment,
                         ))
                 except (ValueError, TypeError, AttributeError) as ex:
                     raise ValueError(
@@ -173,7 +177,7 @@ def __new__(cls, urlstring=None, force_abs=True,
             return tuple.__new__(cls, (
                 scheme, netloc, path,
                 hostname, port, query,
-                username, password
+                username, password, fragment
             ))
 
     def __getattr__(self, name):
@@ -196,15 +200,17 @@ def __repr__(self):
         """
         return (
             "Url(scheme=%r, netloc=%r, path=%r, hostname=%r,"
-            " port=%r, query=%r, username=%r, password=%r)" %
+            " port=%r, query=%r, username=%r, password=%r, fragment=%r)"
+            %
             (self.scheme,
              self.netloc,
              self.path,
              self.hostname,
              self.port,
              self.query,
              self.username,
-             self.password))
+             self.password,
+             self.fragment))
 
     def __str__(self):
         """
@@ -228,14 +234,15 @@ def __str__(self):
         """
         url = self.path
         # XXX: assumes all our URLs are of netloc-type!
-        if self.netloc or (self.scheme and not url.startswith('//')):
-            if url and not url.startswith('/'):
-                url = '/' + url
-            url = '//' + (self.netloc or '') + url
+        if url and not url.startswith('/'):
+            url = '/' + url
+        url = '//' + (self.netloc or '') + url
         if self.scheme:
             url = self.scheme + ':' + url
         if self.query:
             url += '?%s' % self.query
+        if self.fragment:
+            url += '#%s' % self.fragment
         return url
 
     def __eq__(self, other):
@@ -323,7 +330,7 @@ def adjoin(self, relpath):
                    path=os.path.join((self.path or '/'), relpath),
                    hostname=self.hostname, port=self.port,
                    username=self.username, password=self.password,
-                   query=self.query)
+                   query=self.query, fragment=self.fragment)
 
 
 class UrlKeyDict(dict):
@@ -455,7 +462,7 @@ class UrlValueDict(dict):
     URL-type value, regardless of how it was set::
 
         >>> repr(d[1]) == "Url(scheme='file', netloc='', path='/tmp/foo', " \
-        "hostname=None, port=None, query='', username=None, password=None)"
+        "hostname=None, port=None, query='', username=None, password=None, fragment='')"
         True
 
     Class `UrlValueDict` supports initialization by any of the