Merge remote-tracking branch 'origin/release-v1.8' into 2859-fix-the-…

…build

ckan/config/deployment.ini_tmpl

@@ -44,6 +44,11 @@ sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
+## Datastore
+## Uncommment to set the datastore urls
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckantest
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckantest
+
 # repoze.who config
 who.config_file = %(here)s/who.ini
 who.log_level = warning
@@ -198,9 +203,33 @@ ckan.feeds.author_name =
 # If not set, then the value in `ckan.site_url` is used.
 ckan.feeds.author_link =
 
-## Webstore
-## Uncommment to enable datastore
-# ckan.datastore.enabled = 1
+## File Store
+#
+# CKAN allows users to upload files directly to file storage either on the local
+# file system or to online ‘cloud’ storage like Amazon S3 or Google Storage.
+#
+# If you are using local file storage, remember to set ckan.site_url.
+#
+# To enable cloud storage (Google or S3), first run: pip install boto
+#
+# @see http://docs.ckan.org/en/latest/filestore.html
+
+# 'Bucket' to use for file storage
+#ckan.storage.bucket = my-bucket-name
+
+# To enable local file storage:
+#ofs.impl = pairtree
+#ofs.storage_dir = /my/path/to/storage/root/directory
+
+# To enable Google cloud storage:
+#ofs.impl = google
+#ofs.gs_access_key_id =
+#ofs.gs_secret_access_key =
+
+# To enable S3 cloud storage:
+#ofs.impl = s3
+#ofs.aws_access_key_id = ....
+#ofs.aws_secret_access_key = ....
 
 ## ===================================
 ## Extensions

ckan/controllers/group.py

@@ -403,7 +403,8 @@ def history(self, id):
 
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
-                   'schema': self._form_to_db_schema()}
+                   'schema': self._form_to_db_schema(),
+                   'extras_as_string': True}
         data_dict = {'id': id}
         try:
             c.group_dict = get_action('group_show')(context, data_dict)

ckan/controllers/package.py

@@ -788,10 +788,10 @@ def resource_read(self, id, resource_id):
                 get_license_register()[license_id].isopen()
         except KeyError:
             c.package['isopen'] = False
-        c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
-                                    qualified=True)
 
-        c.related_count = c.pkg.related_count
+        #TODO: find a nicer way of doing this
+        c.datastore_api = '%s/api/action' % config.get('ckan.site_url','').rstrip('/')
+
         return render('package/resource_read.html')
 
     def resource_download(self, id, resource_id):

ckan/controllers/storage.py

@@ -262,8 +262,9 @@ def get_metadata(self, label):
         if storage_backend in ['google', 's3']:
             if not label.startswith("/"):
                 label = "/" + label
-            url = "https://%s/%s%s" % (self.ofs.conn.server_name(),
-                                       bucket, label)
+            url = "https://%s%s" % (
+                self.ofs.conn.calling_format.build_host(
+                    self.ofs.conn.server_name(), bucket), label)
         else:
             url = h.url_for('storage_file',
                             label=label,

ckan/lib/cli.py

@@ -13,6 +13,35 @@
 #   i.e. do the imports in methods, after _load_config is called.
 #   Otherwise loggers get disabled.
 
+
+def parse_db_config(config_key='sqlalchemy.url'):
+    ''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+    '''
+    from pylons import config
+    url = config[config_key]
+    regex = [
+        '^\s*(?P<db_type>\w*)',
+        '://',
+        '(?P<db_user>[^:]*)',
+        ':?',
+        '(?P<db_pass>[^@]*)',
+        '@',
+        '(?P<db_host>[^/:]*)',
+        ':?',
+        '(?P<db_port>[^/]*)',
+        '/',
+        '(?P<db_name>[\w.-]*)'
+    ]
+    db_details_match = re.match(''.join(regex), url)
+    if not db_details_match:
+        raise Exception('Could not extract db details from url: %r' % url)
+    db_details = db_details_match.groupdict()
+    return db_details
+
+
 class MockTranslator(object):
     def gettext(self, value):
         return value
@@ -134,14 +163,7 @@ def command(self):
             sys.exit(1)
 
     def _get_db_config(self):
-        from pylons import config
-        url = config['sqlalchemy.url']
-        # e.g. 'postgres://tester:pass@localhost/ckantest3'
-        db_details_match = re.match('^\s*(?P<db_type>\w*)://(?P<db_user>[^:]*):?(?P<db_pass>[^@]*)@(?P<db_host>[^/:]*):?(?P<db_port>[^/]*)/(?P<db_name>[\w.-]*)', url)
-        if not db_details_match:
-            raise Exception('Could not extract db details from url: %r' % url)
-        db_details = db_details_match.groupdict()
-        return db_details
+        return parse_db_config()
 
     def _get_postgres_cmd(self, command):
         self.db_details = self._get_db_config()

ckan/lib/package_saver.py

@@ -32,16 +32,14 @@ def render_package(cls, pkg, context):
         url = pkg.get('url', '')
         c.pkg_url_link = h.link_to(url, url, rel='foaf:homepage', target='_blank') \
                 if url else _("No web page given")
-        if pkg.get('author_email', False):
-            c.pkg_author_link = cls._person_email_link(pkg.get('author', ''), pkg.get('author_email', ''), "Author")
-        else:
-            c.pkg_author_link = _("Author not given")
-        maintainer = pkg.get('maintainer', '')
-        maintainer_email = pkg.get('maintainer_email', '')
-        if maintainer_email:
-            c.pkg_maintainer_link = cls._person_email_link(maintainer, maintainer_email, "Maintainer")
-        else:
-            c.pkg_maintainer_link = _("Maintainer not given")
+
+        c.pkg_author_link = cls._person_email_link(
+                name=pkg.get('author'), email=pkg.get('author_email'),
+                fallback=_("Author not given"))
+        c.pkg_maintainer_link = cls._person_email_link(
+                name=pkg.get('maintainer'), email=pkg.get('maintainer_email'),
+                fallback=_("Maintainer not given"))
+
         c.package_relationships = context['package'].get_relationships_printable()
         c.pkg_extras = []
         for extra in sorted(pkg.get('extras',[]), key=lambda x:x['key']):
@@ -102,9 +100,12 @@ def _revision_validation(cls, log_message):
         return errors
 
     @classmethod
-    def _person_email_link(cls, name, email, reference):
-        assert email
-        return h.mail_to(email_address=email, name=name or email, encode='hex')
+    def _person_email_link(cls, name, email, fallback):
+        if email:
+            return h.mail_to(email_address=email, name=name or email,
+                    encode='hex')
+        else:
+            return name or fallback
 
 class WritePackageFromBoundFieldset(object):
 

ckan/public/scripts/application.js

@@ -1675,8 +1675,8 @@ CKAN.DataPreview = function ($, my) {
     }
 
     // 4 situations
-    // a) webstore_url is active (something was posted to the datastore)
-    // b) csv or xls (but not webstore)
+    // a) something was posted to the datastore - need to check for this
+    // b) csv or xls (but not datastore)
     // c) can be treated as plain text
     // d) none of the above but worth iframing (assumption is
     // that if we got here (i.e. preview shown) worth doing
@@ -1694,9 +1694,12 @@ CKAN.DataPreview = function ($, my) {
       }
     }
 
-    if (resourceData.webstore_url) {
-      resourceData.url = '/api/data/' + resourceData.id;
-      resourceData.backend =  'elasticsearch';
+    // Set recline CKAN backend API endpoint to right location (so it can locate
+    // CKAN DataStore)
+    recline.Backend.Ckan.API_ENDPOINT = CKAN.SITE_URL + '/api';
+
+    if (resourceData.datastore_active) {
+      resourceData.backend =  'ckan';
       var dataset = new recline.Model.Dataset(resourceData);
       var errorMsg = CKAN.Strings.errorLoadingPreview + ': ' + CKAN.Strings.errorDataStore;
       dataset.fetch()