From 039f7755467fd7b4cbfb0dd10b6082288b18e2c6 Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Fri, 20 Apr 2018 13:00:25 -0400
Subject: [PATCH 01/24] [#4162] view filter: use plain queries when required

---
 .../base/javascript/modules/resource-view-filters-form.js | 7 +++++--
 .../base/javascript/modules/resource-view-filters.js      | 8 ++++++--
 .../base/javascript/modules/resource-view-filters-form.js | 7 +++++--
 .../base/javascript/modules/resource-view-filters.js      | 7 +++++--
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
index 2a7cb3f751c..6b9fe7abb71 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
@@ -19,7 +19,6 @@ ckan.module('resource-view-filters-form', function (jQuery) {
                 query;
 
             query = {
-              plain: false,
               resource_id: resourceId,
               limit: queryLimit,
               offset: offset,
@@ -30,7 +29,11 @@ ckan.module('resource-view-filters-form', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              q[filterName] = term + ':*';
+              if (!term.includes(' ')) {
+                term = term + ':*';
+                query.plain = false;
+              }
+              q[filterName] = term;
               query.q = JSON.stringify(q);
             }
 
diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
index 220e4b1abca..fd78092b309 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
@@ -103,7 +103,6 @@ this.ckan.module('resource-view-filters', function (jQuery) {
                 query;
 
             query = {
-              plain: false,
               resource_id: resourceId,
               limit: queryLimit,
               offset: offset,
@@ -112,9 +111,14 @@ this.ckan.module('resource-view-filters', function (jQuery) {
               sort: filterName
             };
 
+
             if (term !== '') {
               var q = {};
-              q[filterName] = term + ':*';
+              if (!term.includes(' ')) {
+                term = term + ':*';
+                query.plain = false;
+              }
+              q[filterName] = term;
               query.q = JSON.stringify(q);
             }
 
diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js
index 2a7cb3f751c..6b9fe7abb71 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js
@@ -19,7 +19,6 @@ ckan.module('resource-view-filters-form', function (jQuery) {
                 query;
 
             query = {
-              plain: false,
               resource_id: resourceId,
               limit: queryLimit,
               offset: offset,
@@ -30,7 +29,11 @@ ckan.module('resource-view-filters-form', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              q[filterName] = term + ':*';
+              if (!term.includes(' ')) {
+                term = term + ':*';
+                query.plain = false;
+              }
+              q[filterName] = term;
               query.q = JSON.stringify(q);
             }
 
diff --git a/ckan/public/base/javascript/modules/resource-view-filters.js b/ckan/public/base/javascript/modules/resource-view-filters.js
index 220e4b1abca..1ee80294557 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters.js
@@ -103,7 +103,6 @@ this.ckan.module('resource-view-filters', function (jQuery) {
                 query;
 
             query = {
-              plain: false,
               resource_id: resourceId,
               limit: queryLimit,
               offset: offset,
@@ -114,7 +113,11 @@ this.ckan.module('resource-view-filters', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              q[filterName] = term + ':*';
+              if (!term.includes(' ')) {
+                term = term + ':*';
+                query.plain = false;
+              }
+              q[filterName] = term;
               query.q = JSON.stringify(q);
             }
 

From cdc5e58f23f2eaf2abb42a85dd3052eae1971163 Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Sun, 6 May 2018 15:35:48 -0400
Subject: [PATCH 02/24] [#4162] view filter: fix endless querying due to
 incorrect total

---
 .../base/javascript/modules/resource-view-filters-form.js       | 2 +-
 .../public-bs2/base/javascript/modules/resource-view-filters.js | 2 +-
 .../base/javascript/modules/resource-view-filters-form.js       | 2 +-
 ckan/public/base/javascript/modules/resource-view-filters.js    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
index 6b9fe7abb71..183026ea96f 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
@@ -41,7 +41,7 @@ ckan.module('resource-view-filters-form', function (jQuery) {
         },
         results: function (data, page) {
           var records = data.result.records,
-              hasMore = (records.length < data.result.total),
+              hasMore = (records.length == queryLimit),
               results;
 
           results = $.map(records, function (record) {
diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
index fd78092b309..beef7bdeb15 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
@@ -126,7 +126,7 @@ this.ckan.module('resource-view-filters', function (jQuery) {
           },
           results: function (data, page) {
             var records = data.result.records,
-                hasMore = (records.length < data.result.total),
+                hasMore = (records.length == queryLimit),
                 results;
 
             results = $.map(records, function (record) {
diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js
index 6b9fe7abb71..183026ea96f 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js
@@ -41,7 +41,7 @@ ckan.module('resource-view-filters-form', function (jQuery) {
         },
         results: function (data, page) {
           var records = data.result.records,
-              hasMore = (records.length < data.result.total),
+              hasMore = (records.length == queryLimit),
               results;
 
           results = $.map(records, function (record) {
diff --git a/ckan/public/base/javascript/modules/resource-view-filters.js b/ckan/public/base/javascript/modules/resource-view-filters.js
index 1ee80294557..2a242311121 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters.js
@@ -125,7 +125,7 @@ this.ckan.module('resource-view-filters', function (jQuery) {
           },
           results: function (data, page) {
             var records = data.result.records,
-                hasMore = (records.length < data.result.total),
+                hasMore = (records.length == queryLimit),
                 results;
 
             results = $.map(records, function (record) {

From 774f31e51c13e82574baa53d58d64dfe18d1159b Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Sun, 6 May 2018 15:46:34 -0400
Subject: [PATCH 03/24] [#4162] view filter: faster query without total calc

---
 .../base/javascript/modules/resource-view-filters-form.js      | 3 ++-
 .../base/javascript/modules/resource-view-filters.js           | 3 ++-
 .../base/javascript/modules/resource-view-filters-form.js      | 3 ++-
 ckan/public/base/javascript/modules/resource-view-filters.js   | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
index 183026ea96f..e0c70cdc791 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
@@ -24,7 +24,8 @@ ckan.module('resource-view-filters-form', function (jQuery) {
               offset: offset,
               fields: filterName,
               distinct: true,
-              sort: filterName
+              sort: filterName,
+              include_total: false
             };
 
             if (term !== '') {
diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
index beef7bdeb15..e6da9f7d785 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
@@ -108,7 +108,8 @@ this.ckan.module('resource-view-filters', function (jQuery) {
               offset: offset,
               fields: filterName,
               distinct: true,
-              sort: filterName
+              sort: filterName,
+              include_total: false
             };
 
 
diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js
index 183026ea96f..e0c70cdc791 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js
@@ -24,7 +24,8 @@ ckan.module('resource-view-filters-form', function (jQuery) {
               offset: offset,
               fields: filterName,
               distinct: true,
-              sort: filterName
+              sort: filterName,
+              include_total: false
             };
 
             if (term !== '') {
diff --git a/ckan/public/base/javascript/modules/resource-view-filters.js b/ckan/public/base/javascript/modules/resource-view-filters.js
index 2a242311121..ac28478bcff 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters.js
@@ -108,7 +108,8 @@ this.ckan.module('resource-view-filters', function (jQuery) {
               offset: offset,
               fields: filterName,
               distinct: true,
-              sort: filterName
+              sort: filterName,
+              include_total: false
             };
 
             if (term !== '') {

From fb18b400ae5db85bad60b75ebf9da35e87d2826e Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Fri, 11 May 2018 09:10:01 -0400
Subject: [PATCH 04/24] [#4162] IE compatibility

---
 .../base/javascript/modules/resource-view-filters-form.js       | 2 +-
 .../public-bs2/base/javascript/modules/resource-view-filters.js | 2 +-
 .../base/javascript/modules/resource-view-filters-form.js       | 2 +-
 ckan/public/base/javascript/modules/resource-view-filters.js    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
index e0c70cdc791..8dd976ce5b1 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters-form.js
@@ -30,7 +30,7 @@ ckan.module('resource-view-filters-form', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              if (!term.includes(' ')) {
+              if (term.indexOf(' ') == -1) {
                 term = term + ':*';
                 query.plain = false;
               }
diff --git a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
index e6da9f7d785..99831075604 100644
--- a/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public-bs2/base/javascript/modules/resource-view-filters.js
@@ -115,7 +115,7 @@ this.ckan.module('resource-view-filters', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              if (!term.includes(' ')) {
+              if (term.indexOf(' ') == -1) {
                 term = term + ':*';
                 query.plain = false;
               }
diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js
index e0c70cdc791..8dd976ce5b1 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters-form.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js
@@ -30,7 +30,7 @@ ckan.module('resource-view-filters-form', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              if (!term.includes(' ')) {
+              if (term.indexOf(' ') == -1) {
                 term = term + ':*';
                 query.plain = false;
               }
diff --git a/ckan/public/base/javascript/modules/resource-view-filters.js b/ckan/public/base/javascript/modules/resource-view-filters.js
index ac28478bcff..5a9ffaecec2 100644
--- a/ckan/public/base/javascript/modules/resource-view-filters.js
+++ b/ckan/public/base/javascript/modules/resource-view-filters.js
@@ -114,7 +114,7 @@ this.ckan.module('resource-view-filters', function (jQuery) {
 
             if (term !== '') {
               var q = {};
-              if (!term.includes(' ')) {
+              if (term.indexOf(' ') == -1) {
                 term = term + ':*';
                 query.plain = false;
               }

From babb752050cb271489cc1f58df22d675fa5f8184 Mon Sep 17 00:00:00 2001
From: Harald von Waldow <harald.vonwaldow@eawag.ch>
Date: Fri, 11 May 2018 17:33:36 +0200
Subject: [PATCH 05/24] autocomplete tagsep - fix

---
 ckan/public/base/javascript/modules/autocomplete.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js
index 8a8f5e9d69c..dbcaf0e2dec 100644
--- a/ckan/public/base/javascript/modules/autocomplete.js
+++ b/ckan/public/base/javascript/modules/autocomplete.js
@@ -2,14 +2,16 @@
  * a list of terms from an API endpoint (provided using data-module-source).
  *
  * source   - A url pointing to an API autocomplete endpoint.
- * interval - The interval between requests in milliseconds (default: 1000).
+ * interval - The interval between requests in milliseconds (default: 300).
  * items    - The max number of items to display (default: 10)
  * tags     - Boolean attribute if true will create a tag input.
  * key      - A string of the key you want to be the form value to end up on
  *            from the ajax returned results
  * label    - A string of the label you want to appear within the dropdown for
  *            returned results
- *
+ * tokensep - An array (passed as string) that contains characters which will
+              be interpreted as separators for tags when typed or pasted
+              (default "[',']").
  * Examples
  *
  *   // <input name="tags" data-module="autocomplete" data-module-source="http://" />
@@ -24,6 +26,7 @@ this.ckan.module('autocomplete', function (jQuery) {
       label: false,
       items: 10,
       source: null,
+      tokensep: [','],
       interval: 300,
       dropdownClass: '',
       containerClass: ''
@@ -50,7 +53,8 @@ this.ckan.module('autocomplete', function (jQuery) {
         formatNoMatches: this.formatNoMatches,
         formatInputTooShort: this.formatInputTooShort,
         dropdownCssClass: this.options.dropdownClass,
-        containerCssClass: this.options.containerClass
+        containerCssClass: this.options.containerClass,
+        tokenSeparators: eval(this.options.tokensep)
       };
 
       // Different keys are required depending on whether the select is

From 56b35b894997527cdae047eeddb86090fa69c6e5 Mon Sep 17 00:00:00 2001
From: Harald von Waldow <harald.vonwaldow@eawag.ch>
Date: Thu, 17 May 2018 20:22:24 +0200
Subject: [PATCH 06/24] separation tags as chars from str.split()

---
 ckan/public/base/javascript/modules/autocomplete.js | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js
index dbcaf0e2dec..562854a7afe 100644
--- a/ckan/public/base/javascript/modules/autocomplete.js
+++ b/ckan/public/base/javascript/modules/autocomplete.js
@@ -9,9 +9,8 @@
  *            from the ajax returned results
  * label    - A string of the label you want to appear within the dropdown for
  *            returned results
- * tokensep - An array (passed as string) that contains characters which will
-              be interpreted as separators for tags when typed or pasted
-              (default "[',']").
+ * tokensep - A string that contains characters which will be interpreted
+ *            as separators for tags when typed or pasted (default ",").
  * Examples
  *
  *   // <input name="tags" data-module="autocomplete" data-module-source="http://" />
@@ -26,7 +25,7 @@ this.ckan.module('autocomplete', function (jQuery) {
       label: false,
       items: 10,
       source: null,
-      tokensep: [','],
+      tokensep: ',',
       interval: 300,
       dropdownClass: '',
       containerClass: ''
@@ -54,7 +53,7 @@ this.ckan.module('autocomplete', function (jQuery) {
         formatInputTooShort: this.formatInputTooShort,
         dropdownCssClass: this.options.dropdownClass,
         containerCssClass: this.options.containerClass,
-        tokenSeparators: eval(this.options.tokensep)
+        tokenSeparators: this.options.tokensep.split('')
       };
 
       // Different keys are required depending on whether the select is

From c8c06c7dab35c503d6e7720699eabd1beb177627 Mon Sep 17 00:00:00 2001
From: Harald von Waldow <harald.vonwaldow@eawag.ch>
Date: Thu, 17 May 2018 20:45:24 +0200
Subject: [PATCH 07/24] adapted test

---
 .../base/test/spec/modules/autocomplete.spec.js      | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ckan/public/base/test/spec/modules/autocomplete.spec.js b/ckan/public/base/test/spec/modules/autocomplete.spec.js
index 93a8fb57a54..9e8fffa0469 100644
--- a/ckan/public/base/test/spec/modules/autocomplete.spec.js
+++ b/ckan/public/base/test/spec/modules/autocomplete.spec.js
@@ -65,7 +65,8 @@ describe('ckan.modules.AutocompleteModule()', function () {
         formatNoMatches: this.module.formatNoMatches,
         formatInputTooShort: this.module.formatInputTooShort,
         createSearchChoice: this.module.formatTerm, // Not used by tags.
-        initSelection: this.module.formatInitialValue
+        initSelection: this.module.formatInitialValue,
+	tokenSeparators: [',']
       });
     });
 
@@ -82,7 +83,8 @@ describe('ckan.modules.AutocompleteModule()', function () {
         formatResult: this.module.formatResult,
         formatNoMatches: this.module.formatNoMatches,
         formatInputTooShort: this.module.formatInputTooShort,
-        initSelection: this.module.formatInitialValue
+        initSelection: this.module.formatInitialValue,
+	tokenSeparators: [',']
       });
 
       it('should watch the keydown event on the select2 input');
@@ -100,7 +102,8 @@ describe('ckan.modules.AutocompleteModule()', function () {
           formatResult: this.module.formatResult,
           formatNoMatches: this.module.formatNoMatches,
           formatInputTooShort: this.module.formatInputTooShort,
-          initSelection: this.module.formatInitialValue
+          initSelection: this.module.formatInitialValue,
+	  tokenSeparators: [',']
         });
       });
 
@@ -117,7 +120,8 @@ describe('ckan.modules.AutocompleteModule()', function () {
           formatResult: this.module.formatResult,
           formatNoMatches: this.module.formatNoMatches,
           formatInputTooShort: this.module.formatInputTooShort,
-          initSelection: this.module.formatInitialValue
+          initSelection: this.module.formatInitialValue,
+	  tokenSeparators: [',']
         });
       });
 

From ce829fa6c34c2670c1df7004cd349cd42459d9b7 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Wed, 23 May 2018 17:08:47 +0200
Subject: [PATCH 08/24] Get file from request.files

---
 ckan/views/admin.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ckan/views/admin.py b/ckan/views/admin.py
index 3a80ffbd56d..9f97a90e4ba 100644
--- a/ckan/views/admin.py
+++ b/ckan/views/admin.py
@@ -116,6 +116,10 @@ def post(self):
                         logic.parse_params(
                             request.form, ignore_keys=CACHE_PARAMETERS))))
             del data_dict['save']
+            if not request.files:
+                base.abort(404, _("Upload file not found"))
+
+            data_dict.update(request.files.to_dict())
             data = logic.get_action(u'config_option_update')({
                 u'user': g.user
             }, data_dict)

From 9c03d1a81ec8372e40ab25d47e447d7788ae5e1a Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Wed, 23 May 2018 17:50:42 +0200
Subject: [PATCH 09/24] remove wrongly added condition

---
 ckan/views/admin.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/ckan/views/admin.py b/ckan/views/admin.py
index 9f97a90e4ba..39148744047 100644
--- a/ckan/views/admin.py
+++ b/ckan/views/admin.py
@@ -115,11 +115,8 @@ def post(self):
                     logic.tuplize_dict(
                         logic.parse_params(
                             request.form, ignore_keys=CACHE_PARAMETERS))))
-            del data_dict['save']
-            if not request.files:
-                base.abort(404, _("Upload file not found"))
-
             data_dict.update(request.files.to_dict())
+            del data_dict['save']
             data = logic.get_action(u'config_option_update')({
                 u'user': g.user
             }, data_dict)

From f8a8d7420552628c016eabd717f461b5466894a0 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Thu, 24 May 2018 18:55:54 +0200
Subject: [PATCH 10/24] merge request dicts

---
 ckan/views/admin.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ckan/views/admin.py b/ckan/views/admin.py
index 39148744047..defd3850f17 100644
--- a/ckan/views/admin.py
+++ b/ckan/views/admin.py
@@ -110,12 +110,14 @@ def get(self):
 
     def post(self):
         try:
+            req = request.form.copy()
+            req.update(request.files.to_dict())
             data_dict = logic.clean_dict(
                 dict_fns.unflatten(
                     logic.tuplize_dict(
                         logic.parse_params(
-                            request.form, ignore_keys=CACHE_PARAMETERS))))
-            data_dict.update(request.files.to_dict())
+                            req, ignore_keys=CACHE_PARAMETERS))))
+
             del data_dict['save']
             data = logic.get_action(u'config_option_update')({
                 u'user': g.user

From 9c6a2d037dd98179a23b71ada08726efb83502de Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Wed, 9 May 2018 11:15:08 +0200
Subject: [PATCH 11/24] Update changelog

---
 CHANGELOG.rst | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 115a036c1b7..23ffcd07945 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -28,6 +28,102 @@ Changes and deprecations:
  * `ckan.recaptcha.version` config option is removed, since v2 is the only valid
     version now (#4061)
 
+v.2.8.0 2018-05-09
+==================
+
+Major fixes:
+ * CKAN API documentation (#2944)
+ * Foundation work for common Flask / Pylons objects and functions (#3196)
+ * [#3196] i18n for Flask endpoints (#3213)
+ * Migrate API controller to Flask blueprint (#3229)
+ * Port feeds controller to Flask blueprint (#3567)
+ * Datastore upsert performance (#3556)
+ * Remove old Celery-based tasks (#4055)
+
+Minor fixes:
+ * Type override for next datapusher run (#3557)
+ * Problems in background workers with non-core database relations (#3606)
+ * Minor issues with new Bootstrap 3 default theme (#3678)
+ * Overriding datastore authorization system (#3679)
+ * Retire Google+ social network link (#3680)
+ * Standardize on url_for (#3831)
+ * Deprecate notify_after_commit (#3633)
+ *  _mail_recipient header override (#3781)
+ * Restrict access to forms (#3684)
+ * Bootstrap3 UI issues (#3754)
+ * Clean up template rendering code (#3923)
+ * Permission labels are indexed by type text in SOLR (#3863)
+ * CLI commands require a Flask test request context (#3760)
+ * Improve IGroupForm support (#4031)
+ * Allow IValidator to override existing validators (#3865)
+ * Shrink datastore_create response size (#3810)
+ * Stable version URLs CKAN for documentation (#4209)
+ * API Documentation update (#4136)
+ * Documentation of Data Dictionary  (#3989)
+ * Remove datastore legacy mode (#4041)
+ * File uploads don't work on new Flask based API (#3869)
+ * Datapusher extension: Custom url instead of ckan site url (#4013)
+ * {% ckan_extends %} not working on templates served by Flask (#4044)
+
+
+
+Bug fixes:
+ * Render_datetime can't handle dates before year 1900 (#2228)
+ * DatapusherPlugin implementation of notify() can call 'datapusher_submit' multiple times (#2334)
+ * Dataset creation page generates incorrect URLs with Chrome autocomplete (#2501)
+ * Search buttons need accessible labels (#2550)
+ * Column name length limit for datastore upload (#2804)
+ * #2373: Do not validate packages or resources from database to views (#3016)
+ * Creation of dataset - different behaviour between Web API & CKAN Interface functionality (#3528)
+ * Redirecting to same page in non-root hosted ckan adds extra root_path to url  (#3499)
+ * Beaker 1.8.0 exception when the code is served from OSX via Vagrant (#3512)
+ * Allow datastore_search_sql on private datasets (#2562)
+ * Add "Add Dataset" button to user's and group's page (#2794)
+ * Some links in CKAN is not reachable (#2898)
+ * Exception when specifying a directory in the ckan.i18n_directory option (#3539)
+ * Resource view filter user filters JS error (#3590)
+ * Recaptcha v1 will stop working 2018-3-31 (#4061)
+ * "Testing coding standards" page in docs is missing code snippets (#3635)
+ * Followers count not updated immediately on UI (#3639)
+ * jQuery version (#3665)
+ * Search icon on many pages is not properly vertically aligned (#3654)
+ * Datatables view can't be used as a default view (#3669)
+ * Resource URL is not validated on create/update (#3660)
+ * Upload to Datastore tab shows incorrect time at Upload Log (#3588)
+ * Filter results button is not working (#3593)
+ * Broken link in "Upgrading CKAN’s dependencies" doc page (#3637)
+ * Default logo image not properly saved (#3656)
+ * Activity test relies on datetime.now() (#3644)
+ * Info block text for Format field not properly aligned in resource form page (#3663)
+ * Issue upon creating new organization/group through UI form (#3661)
+ * In API docs "package_create" lists "owner_org" as optional (#3647)
+ * Embed modal window not working (#3731)
+ * Frontent build command does not work on master (#3688)
+ * Loading image duplicated  (#3716)
+ * Support setuptools 36.x (#3738)
+ * Datastore set-up error - logging getting in the way (#3694)
+ * Registering a new account redirects to an unprefixed url (#3834)
+ * Exception in search page when not authorized (#4081)
+ * Datastore full-text-search column is populated by postgres trigger rather than python (#3785)
+ * Datastore dump results are not the same as data in database (#4150)
+ * Adding filter at resoruce preview doesn't work while site is setup with ckan.root_path param (#4140)
+ * No such file or directory: '/usr/lib/ckan/default/src/ckan/requirement-setuptools.txt' during installation from source (#3641)
+ * Register user form missing required field indicators (#3658)
+ * Datastore full-text-search column is populated by postgres trigger rather than python  (#3786)
+ * Add missing major changes to change log (#3799)
+ * Paster/CLI config-tool requires _get_test_app which in turn requires a dev-only dependency (#3806)
+ * Change log doesn't mention necessary Solr scheme upgrade (#3851)
+ * CKAN 2.7 header not showing toggle bars (#3880)
+ * TypeError: expected byte string object, value of type unicode found (#3921)
+ * CKAN's state table clashes with PostGIS generated TIGER state table (#3929)
+ * [Docker] entrypoint initdb.d sql files copied to root (#3939)
+ * DataStore status page throws TypeError - Bleach upgrade regression (#3968)
+ * Source install error with who.ini (#4020)
+ * making a JSONP call to the CKAN API returns the wrong mime type (#4022)
+ * Deleting a resource sets datastore_active=False to all resources and overrides their extras (#4042)
+ * Deleting first Group and Organization custom field is not possible (#4094)
+
+
 v2.7.3 2018-03-15
 =================
 

From fbf364d973527aa576364543aba21877e23e621d Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Wed, 9 May 2018 13:27:28 +0200
Subject: [PATCH 12/24] Changelog tweaks

---
 CHANGELOG.rst | 78 ++++++++++++++++++++++++++-------------------------
 1 file changed, 40 insertions(+), 38 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 23ffcd07945..b1b62e2b3a6 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -7,11 +7,15 @@
 Changelog
 ---------
 
-v?? (TBA)
-=========
+v.2.8.0 2018-05-09
+==================
 
-Note: This version requires re-running the 'datastore set-permissions' command
-   (assuming you run DataStore). See: :ref:`datastore-set-permissions`
+General notes:
+ * This version requires a requirements upgrade on source installations
+ * This version requires a database upgrade
+ * This version requires a Solr schema upgrade
+ * This version requires re-running the ``datastore set-permissions`` command
+   (assuming you are using the DataStore). See: :ref:`datastore-set-permissions`
 
    Otherwise new and updated datasets will not be searchable in DataStore and
    the logs will contain this error::
@@ -21,53 +25,39 @@ Note: This version requires re-running the 'datastore set-permissions' command
    CKAN developers should also re-run set-permissions on the test database:
    :ref:`datastore-test-set-permissions`
 
-Changes and deprecations:
- * The old Celery based background jobs have been removed in CKAN 2.8 in favour of the new RQ based
-   jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
-   of course use Celery but they will need to handle the management themselves.
- * `ckan.recaptcha.version` config option is removed, since v2 is the only valid
-    version now (#4061)
+ * There are several old features being officially deprecated starting from
+   this version. Check the *Deprecations* section to be prepared.
 
-v.2.8.0 2018-05-09
-==================
+Major changes:
+ * New revamped frontend templates based on Bootstrap 3, see "Changes and deprecations" (#3547) 
+ * Allow datastore_search_sql on private datasets (#2562)
+ * New Flask blueprints migrated from old Pylons controllers: user, dashboard, feeds, admin and home (#3927, #3870, #3775, #3762)
+ * Improved support for custom groups and organization types (#4032)
+ * Hide user details to anonymous users (#3915)
 
-Major fixes:
- * CKAN API documentation (#2944)
- * Foundation work for common Flask / Pylons objects and functions (#3196)
- * [#3196] i18n for Flask endpoints (#3213)
- * Migrate API controller to Flask blueprint (#3229)
- * Port feeds controller to Flask blueprint (#3567)
- * Datastore upsert performance (#3556)
- * Remove old Celery-based tasks (#4055)
-
-Minor fixes:
- * Type override for next datapusher run (#3557)
- * Problems in background workers with non-core database relations (#3606)
- * Minor issues with new Bootstrap 3 default theme (#3678)
+Minor changes:
+ * Allow chaining of authentication functions (#3679)
+ * Show custom dataset types in search pages (#3807)
  * Overriding datastore authorization system (#3679)
- * Retire Google+ social network link (#3680)
  * Standardize on url_for (#3831)
  * Deprecate notify_after_commit (#3633)
  *  _mail_recipient header override (#3781)
- * Restrict access to forms (#3684)
- * Bootstrap3 UI issues (#3754)
+ * Restrict access to member forms (#3684)
  * Clean up template rendering code (#3923)
  * Permission labels are indexed by type text in SOLR (#3863)
  * CLI commands require a Flask test request context (#3760)
- * Improve IGroupForm support (#4031)
  * Allow IValidator to override existing validators (#3865)
  * Shrink datastore_create response size (#3810)
  * Stable version URLs CKAN for documentation (#4209)
  * API Documentation update (#4136)
  * Documentation of Data Dictionary  (#3989)
  * Remove datastore legacy mode (#4041)
- * File uploads don't work on new Flask based API (#3869)
- * Datapusher extension: Custom url instead of ckan site url (#4013)
- * {% ckan_extends %} not working on templates served by Flask (#4044)
-
-
+ * Map old Pylons routes to Flask ones (#4066)
 
 Bug fixes:
+ * File uploads don't work on new Flask based API (#3869)
+ * {% ckan_extends %} not working on templates served by Flask (#4044)
+ * Problems in background workers with non-core database relations (#3606)
  * Render_datetime can't handle dates before year 1900 (#2228)
  * DatapusherPlugin implementation of notify() can call 'datapusher_submit' multiple times (#2334)
  * Dataset creation page generates incorrect URLs with Chrome autocomplete (#2501)
@@ -77,7 +67,6 @@ Bug fixes:
  * Creation of dataset - different behaviour between Web API & CKAN Interface functionality (#3528)
  * Redirecting to same page in non-root hosted ckan adds extra root_path to url  (#3499)
  * Beaker 1.8.0 exception when the code is served from OSX via Vagrant (#3512)
- * Allow datastore_search_sql on private datasets (#2562)
  * Add "Add Dataset" button to user's and group's page (#2794)
  * Some links in CKAN is not reachable (#2898)
  * Exception when specifying a directory in the ckan.i18n_directory option (#3539)
@@ -85,7 +74,7 @@ Bug fixes:
  * Recaptcha v1 will stop working 2018-3-31 (#4061)
  * "Testing coding standards" page in docs is missing code snippets (#3635)
  * Followers count not updated immediately on UI (#3639)
- * jQuery version (#3665)
+ * Increase jQuery version (#3665)
  * Search icon on many pages is not properly vertically aligned (#3654)
  * Datatables view can't be used as a default view (#3669)
  * Resource URL is not validated on create/update (#3660)
@@ -100,7 +89,6 @@ Bug fixes:
  * Embed modal window not working (#3731)
  * Frontent build command does not work on master (#3688)
  * Loading image duplicated  (#3716)
- * Support setuptools 36.x (#3738)
  * Datastore set-up error - logging getting in the way (#3694)
  * Registering a new account redirects to an unprefixed url (#3834)
  * Exception in search page when not authorized (#4081)
@@ -113,7 +101,6 @@ Bug fixes:
  * Add missing major changes to change log (#3799)
  * Paster/CLI config-tool requires _get_test_app which in turn requires a dev-only dependency (#3806)
  * Change log doesn't mention necessary Solr scheme upgrade (#3851)
- * CKAN 2.7 header not showing toggle bars (#3880)
  * TypeError: expected byte string object, value of type unicode found (#3921)
  * CKAN's state table clashes with PostGIS generated TIGER state table (#3929)
  * [Docker] entrypoint initdb.d sql files copied to root (#3939)
@@ -123,6 +110,21 @@ Bug fixes:
  * Deleting a resource sets datastore_active=False to all resources and overrides their extras (#4042)
  * Deleting first Group and Organization custom field is not possible (#4094)
 
+Changes and deprecations:
+ * The default templates included in CKAN core have been updated to use Bootstrap 3. Extensions
+   implementing custom themes are encouraged to update their templates, but they can still
+   make CKAN load the old Bootstrap 2 templates during the transition using the following
+   configuration options::
+
+        ckan.base_public_folder = public-bs2
+        ckan.base_templates_folder = templates-bs2
+
+ * The API versions 1 and 2 (also known as the REST API), ie ``/api/rest/*`` have been
+   completely removed in favour of the version 3 (action API, ``/api/action/*``).
+ * The old Celery based background jobs have been removed in CKAN 2.8 in favour of the new RQ based
+   jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
+   of course use Celery but they will need to handle the management themselves.
+ * The ``ckan.recaptcha.version`` config option is now removed, since v2 is the only valid version now (#4061)
 
 v2.7.3 2018-03-15
 =================

From f81bdfcd3c4af734bd49ce823b706fdac76eeab5 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Tue, 8 May 2018 17:18:27 +0200
Subject: [PATCH 13/24] update changelog

---
 CHANGELOG.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index b1b62e2b3a6..a17493244ef 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -126,6 +126,12 @@ Changes and deprecations:
    of course use Celery but they will need to handle the management themselves.
  * The ``ckan.recaptcha.version`` config option is now removed, since v2 is the only valid version now (#4061)
 
+v2.7.4 2018-05-09
+=================
+
+ * Adding filter at resoruce preview doesn't work while site is setup with ckan.root_path param (#4140)
+ * Datastore dump results are not the same as data in database (#4150)
+
 v2.7.3 2018-03-15
 =================
 

From 76b993b84cfc16473a49d645e7fce0cce76f0603 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Mon, 28 May 2018 09:54:29 +0200
Subject: [PATCH 14/24] Update changelog dev-v2.6

---
 CHANGELOG.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index a17493244ef..e4717c06c4d 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -29,7 +29,7 @@ General notes:
    this version. Check the *Deprecations* section to be prepared.
 
 Major changes:
- * New revamped frontend templates based on Bootstrap 3, see "Changes and deprecations" (#3547) 
+ * New revamped frontend templates based on Bootstrap 3, see "Changes and deprecations" (#3547)
  * Allow datastore_search_sql on private datasets (#2562)
  * New Flask blueprints migrated from old Pylons controllers: user, dashboard, feeds, admin and home (#3927, #3870, #3775, #3762)
  * Improved support for custom groups and organization types (#4032)
@@ -309,6 +309,13 @@ Deprecations:
    jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
    of course use Celery but they will need to handle the management themselves.
 
+v2.6.6 2018-05-09
+=================
+
+* Adding filter at resoruce preview doesn't work while site is setup with ckan.root_path param (#4140)
+* Stable version URLs CKAN for documentation (#4209)
+* Add Warning in docs sidebar (#4209)
+
 v2.6.5 2018-03-15
 =================
 

From 6be86af59046cffc2b6fbf6b6a2f06700a6950d8 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Mon, 28 May 2018 10:00:06 +0200
Subject: [PATCH 15/24] Update changelog for release-v.2.5.9

---
 CHANGELOG.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index e4717c06c4d..957c02a3ce3 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -487,6 +487,13 @@ Bug fixes:
 API changes and deprecations:
  * Replace `c.__version__` with new helper `h.ckan_version()` (`#3103 <https://github.com/ckan/ckan/pull/3103>`_)
 
+v2.5.9 2018-05-09
+=================
+
+* Adding filter at resoruce preview doesn't work while site is setup with ckan.root_path param (#4140)
+* Add Warning in docs sidebar (#4209)
+* Point API docs to stable URL (#4209)
+
 v2.5.8 2018-03-15
 =================
 

From feaf3d339bad976f607558ea516cb0dc4cc79853 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Tue, 5 Jun 2018 16:56:10 +0200
Subject: [PATCH 16/24] use current version for older releases

---
 doc/conf.py                                   | 36 +++++++++++++++++--
 .../install-from-docker-compose.rst           |  2 +-
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/doc/conf.py b/doc/conf.py
index a1936b6e52c..f9c4a71688f 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -183,10 +183,26 @@ def get_status_of_this_version():
         return 'unsupported'
 
 
+def get_current_release_tag():
+    ''' Return the name of the tag for the current release
+
+    e.g.: "ckan-2.7.4"
+
+    '''
+    release_tags_ = get_release_tags()
+
+    current_tag = "ckan-{}".format(version)
+
+    if release_tags_.__contains__(current_tag):
+        return current_tag
+    else:
+        return 'COULD_NOT_DETECT_TAG_VERSION'
+
+
 def get_latest_release_tag():
     '''Return the name of the git tag for the latest stable release.
 
-    e.g.: "ckan-2.1.1"
+    e.g.: "ckan-2.7.4"
 
     This requires git to be installed.
 
@@ -212,6 +228,19 @@ def get_latest_release_version():
     return version
 
 
+def get_current_release_version():
+    '''Return the version number of the current release.
+
+    e.g. "2.1.1"
+
+    '''
+    version = get_current_release_tag()[len('ckan-'):]
+
+    # TODO: We could assert here that latest_version matches X.Y.Z.
+
+    return version
+
+
 def get_latest_package_name(distro='trusty'):
     '''Return the filename of the Ubuntu package for the latest stable release.
 
@@ -266,7 +295,8 @@ def write_substitutions_file(**kwargs):
             f.write('.. |{name}| replace:: {substitution}\n'.format(
                     name=name, substitution=substitution))
 
-
+current_release_tag = get_current_release_tag()
+current_release_version = get_current_release_version()
 latest_release_tag_value = get_latest_release_tag()
 latest_release_version = get_latest_release_version()
 latest_minor_version = latest_release_version[:3]
@@ -276,7 +306,7 @@ def write_substitutions_file(**kwargs):
 
 write_substitutions_file(
     latest_release_tag=latest_release_tag_value,
-    latest_release_version=get_latest_release_version(),
+    latest_release_version=latest_release_version,
     latest_package_name_precise=get_latest_package_name('precise'),
     latest_package_name_trusty=get_latest_package_name('trusty'),
     latest_package_name_xenial=get_latest_package_name('xenial'),
diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst
index dfdbbe4893a..bef21d2f7f5 100644
--- a/doc/maintaining/installing/install-from-docker-compose.rst
+++ b/doc/maintaining/installing/install-from-docker-compose.rst
@@ -71,7 +71,7 @@ Clone CKAN into a directory of your choice::
 This will use the latest CKAN master, which may not be stable enough for production use.
 To use a stable version, checkout the respective tag, e.g.::
 
-    git checkout tags/ckan-2.6.2
+    git checkout tags/|latest_release_tag|
 
 ----------------------
 2. Build Docker images

From d3ad3bc74573965268539be1710e7f9b6ddfb0c6 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Tue, 5 Jun 2018 22:15:35 +0200
Subject: [PATCH 17/24] Set tag with variable

---
 doc/maintaining/installing/install-from-docker-compose.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst
index bef21d2f7f5..9c2fca271ec 100644
--- a/doc/maintaining/installing/install-from-docker-compose.rst
+++ b/doc/maintaining/installing/install-from-docker-compose.rst
@@ -69,9 +69,11 @@ Clone CKAN into a directory of your choice::
     git clone https://github.com/ckan/ckan.git
 
 This will use the latest CKAN master, which may not be stable enough for production use.
-To use a stable version, checkout the respective tag, e.g.::
+To use a stable version, checkout the respective tag, e.g.:
 
-    git checkout tags/|latest_release_tag|
+    .. parsed-literal::
+
+        git checkout tags/|latest_release_tag|
 
 ----------------------
 2. Build Docker images

From 5a306eda067116376b7fa9706b4642811398f8a8 Mon Sep 17 00:00:00 2001
From: Ross Jones <ross@mailbolt.com>
Date: Wed, 6 Jun 2018 09:05:51 +0100
Subject: [PATCH 18/24] Include requirement-setuptools.txt in manifest

When attempting to `pip install ckan` it fails as the setup.py requires
the requirement-setuptools.txt to be present, and it is not.  This
change makes sure that the pip install will not fail due to the absence
of this file.

This PR does not include the requirements.txt as this would likely
require further discussion about how requirements are loaded.

Fixes #4271
---
 MANIFEST.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 37e122f5903..6f606aeca42 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -19,4 +19,4 @@ include CHANGELOG.txt
 include ckan/migration/migrate.cfg
 include ckan/migration/README
 recursive-include ckan/migration/versions *.sql
-recursive-include ckan_deb *
+include requirement-setuptools.txt

From 87ab28d0f75732914ca6e5575822d8df504b948d Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Wed, 6 Jun 2018 11:41:22 +0200
Subject: [PATCH 19/24] fix icon

---
 ckan/templates-bs2/snippets/license.html | 2 +-
 ckan/templates/snippets/license.html     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ckan/templates-bs2/snippets/license.html b/ckan/templates-bs2/snippets/license.html
index 364dfa9f32f..85ce763a7d0 100644
--- a/ckan/templates-bs2/snippets/license.html
+++ b/ckan/templates-bs2/snippets/license.html
@@ -18,7 +18,7 @@
       {% block license_wrapper %}
         <section class="module module-narrow module-shallow license">
           {% block license_title %}
-            <h2 class="module-heading"><i class="fa fa-medium fa-lock"></i> {{ _('License') }}</h2>
+            <h2 class="module-heading"><i class="fa fa-lock"></i> {{ _('License') }}</h2>
           {% endblock %}
           {% block license_content %}
             <p class="module-content">
diff --git a/ckan/templates/snippets/license.html b/ckan/templates/snippets/license.html
index 364dfa9f32f..85ce763a7d0 100644
--- a/ckan/templates/snippets/license.html
+++ b/ckan/templates/snippets/license.html
@@ -18,7 +18,7 @@
       {% block license_wrapper %}
         <section class="module module-narrow module-shallow license">
           {% block license_title %}
-            <h2 class="module-heading"><i class="fa fa-medium fa-lock"></i> {{ _('License') }}</h2>
+            <h2 class="module-heading"><i class="fa fa-lock"></i> {{ _('License') }}</h2>
           {% endblock %}
           {% block license_content %}
             <p class="module-content">

From af9afa90579f22b2a27520e619f0d84ad3adaf8c Mon Sep 17 00:00:00 2001
From: Goce Mitevski <goce.mitevski@keitaro.com>
Date: Thu, 7 Jun 2018 10:22:52 +0200
Subject: [PATCH 20/24] Fix issue #4273

---
 ckan/public/base/less/masthead.less | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ckan/public/base/less/masthead.less b/ckan/public/base/less/masthead.less
index 791deb32185..d3c19b4d68d 100644
--- a/ckan/public/base/less/masthead.less
+++ b/ckan/public/base/less/masthead.less
@@ -119,9 +119,6 @@
     .section {
         float: left;
     }
-    input[type="text"] {
-        border-color: darken(@mastheadBackgroundColor, 5);
-    }
     .navigation {
         &.section {
             @media (min-width: @screen-sm-min) and (max-width: @screen-sm-max) {

From 1b9f5655de08e059ecaea67e916ddb6f8d2acb2c Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Thu, 7 Jun 2018 10:43:40 +0200
Subject: [PATCH 21/24] remove storage controller

---
 ckan/config/routing.py              |  4 --
 ckan/controllers/storage.py         | 98 -----------------------------
 ckan/tests/test_coding_standards.py |  1 -
 3 files changed, 103 deletions(-)
 delete mode 100644 ckan/controllers/storage.py

diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 60209d13692..432c3822454 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -309,10 +309,6 @@ def make_map():
         m.connect('/revision/list', action='list')
         m.connect('/revision/{id}', action='read')
 
-    with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m:
-        m.connect('storage_file', '/storage/f/{label:.*}',
-                  action='file')
-
     with SubMapper(map, controller='util') as m:
         m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
         m.connect('/util/redirect', action='redirect')
diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py
deleted file mode 100644
index 536bef679e2..00000000000
--- a/ckan/controllers/storage.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# encoding: utf-8
-
-'''
-
-Note: This is the old file store controller for CKAN < 2.2.
-If you are looking for how the file uploads work, you should check
-`lib/uploader.py` and the `resource_download` method of the package
-controller.
-
-'''
-import os
-import re
-
-from ofs import get_impl
-from paste.fileapp import FileApp
-
-from ckan.lib.base import BaseController, request, config, h, abort
-
-
-from logging import getLogger
-log = getLogger(__name__)
-
-
-BUCKET = config.get('ckan.storage.bucket', 'default')
-key_prefix = config.get('ckan.storage.key_prefix', 'file/')
-
-_eq_re = re.compile(r"^(.*)(=[0-9]*)$")
-
-
-def create_pairtree_marker(folder):
-    """ Creates the pairtree marker for tests if it doesn't exist """
-    if not folder[:-1] == '/':
-        folder = folder + '/'
-
-    directory = os.path.dirname(folder)
-    if not os.path.exists(directory):
-        os.makedirs(directory)
-
-    target = os.path.join(directory, 'pairtree_version0_1')
-    if os.path.exists(target):
-        return
-
-    open(target, 'wb').close()
-
-
-def get_ofs():
-    """Return a configured instance of the appropriate OFS driver.
-    """
-    storage_backend = config['ofs.impl']
-    kw = {}
-    for k, v in config.items():
-        if not k.startswith('ofs.') or k == 'ofs.impl':
-            continue
-        kw[k[4:]] = v
-
-    # Make sure we have created the marker file to avoid pairtree issues
-    if storage_backend == 'pairtree' and 'storage_dir' in kw:
-        create_pairtree_marker(kw['storage_dir'])
-
-    ofs = get_impl(storage_backend)(**kw)
-    return ofs
-
-
-class StorageController(BaseController):
-    '''Upload to storage backend.
-    '''
-    _ofs_impl = None
-
-    @property
-    def ofs(self):
-        if not StorageController._ofs_impl:
-            StorageController._ofs_impl = get_ofs()
-        return StorageController._ofs_impl
-
-    def file(self, label):
-        exists = self.ofs.exists(BUCKET, label)
-        if not exists:
-            # handle erroneous trailing slash by redirecting to url w/o slash
-            if label.endswith('/'):
-                label = label[:-1]
-                # This may be best being cached_url until we have moved it into
-                # permanent storage
-                file_url = h.url_for('storage_file', label=label)
-                h.redirect_to(file_url)
-            else:
-                abort(404)
-
-        file_url = self.ofs.get_url(BUCKET, label)
-        if file_url.startswith("file://"):
-            metadata = self.ofs.get_metadata(BUCKET, label)
-            filepath = file_url[len("file://"):]
-            headers = {
-                # 'Content-Disposition':'attachment; filename="%s"' % label,
-                'Content-Type': metadata.get('_format', 'text/plain')}
-            fapp = FileApp(filepath, headers=None, **headers)
-            return fapp(request.environ, self.start_response)
-        else:
-            h.redirect_to(file_url.encode('ascii', 'ignore'))
diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
index b8ef315eb31..4f83f9dcd75 100644
--- a/ckan/tests/test_coding_standards.py
+++ b/ckan/tests/test_coding_standards.py
@@ -260,7 +260,6 @@ def find_unprefixed_string_literals(filename):
     u'ckan/controllers/package.py',
     u'ckan/controllers/partyline.py',
     u'ckan/controllers/revision.py',
-    u'ckan/controllers/storage.py',
     u'ckan/controllers/tag.py',
     u'ckan/controllers/user.py',
     u'ckan/controllers/util.py',

From 1ca5becb28d28517d99a57244f1a44079c10e264 Mon Sep 17 00:00:00 2001
From: Konstantin Sivakov <konstantin.sivakov@gmail.com>
Date: Thu, 7 Jun 2018 14:03:33 +0200
Subject: [PATCH 22/24] small fix for #3991

---
 doc/maintaining/installing/install-from-docker-compose.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst
index 9c2fca271ec..ecf6f915f33 100644
--- a/doc/maintaining/installing/install-from-docker-compose.rst
+++ b/doc/maintaining/installing/install-from-docker-compose.rst
@@ -175,7 +175,7 @@ a. Create and configure datastore database
 
 With running CKAN containers, execute the built-in setup scripts against the ``db`` container::
 
-    docker exec -it db psql -U ckan -f 00_create_datastore.sql
+    docker exec -it db psql -U ckan -f 00_create_datastore.sh
     docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan datastore set-permissions -c /etc/ckan/production.ini | docker exec -i db psql -U ckan
 
 The first script will create the datastore database and the datastore readonly user in the ``db``

From 716a1ea652d4fcd00ea777825a6c98f6b916eadc Mon Sep 17 00:00:00 2001
From: Ross Jones <ross@mailbolt.com>
Date: Thu, 7 Jun 2018 15:14:11 +0100
Subject: [PATCH 23/24] Replace HTTPIE examples with CURL examples

As per #2293 we replace the httpie examples with the equivalent in curl.
It appears that the examples were using http and therefore were getting
a 302 to the https url, and as we all know to our pain, POST data
doesn't follow redirects and so the examples weren't always working.
Have changes some occurrences of http://demo.ckan.org to
https://demo.ckan.org
---
 doc/api/index.rst                       | 18 +++++++++---------
 doc/contributing/documentation.rst      |  2 +-
 doc/maintaining/linked-data-and-rdf.rst |  8 ++++----
 doc/maintaining/stats.rst               |  2 +-
 doc/user-guide.rst                      |  2 +-
 5 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/doc/api/index.rst b/doc/api/index.rst
index 23c86f595d2..8f42754441d 100644
--- a/doc/api/index.rst
+++ b/doc/api/index.rst
@@ -73,13 +73,13 @@ To call the CKAN API, post a JSON dictionary in an HTTP POST request to one of
 CKAN's API URLs. The parameters for the API function should be given in the
 JSON dictionary. CKAN will also return its response in a JSON dictionary.
 
-One way to post a JSON dictionary to a URL is using the command-line HTTP
-client `HTTPie <http://httpie.org/>`_.  For example, to get a list of the names
+One way to post a JSON dictionary to a URL is using the command-line 
+client `Curl <https://curl.haxx.se/>`_.  For example, to get a list of the names
 of all the datasets in the ``data-explorer`` group on demo.ckan.org, install
-HTTPie and then call the ``group_list`` API function by running this command
+curl and then call the ``group_list`` API function by running this command
 in a terminal::
 
-    http http://demo.ckan.org/api/3/action/group_list
+    curl http://demo.ckan.org/api/3/action/group_list
 
 The response from CKAN will look like this::
 
@@ -262,16 +262,16 @@ can be configured with the ``apikey_header_name`` option in your CKAN
 configuration file.)
 
 For example, to ask whether or not you're currently following the user
-``markw`` on demo.ckan.org using HTTPie, run this command::
+``markw`` on demo.ckan.org using curl, run this command::
 
-    http http://demo.ckan.org/api/3/action/am_following_user id=markw Authorization:XXX
+    curl -H "Authorization: XXX"  https://demo.ckan.org/api/3/action/am_following_user?id=markw 
 
 (Replacing ``XXX`` with your API key.)
 
 Or, to get the list of activities from your user dashboard on demo.ckan.org,
 run this Python code::
 
-    request = urllib2.Request('http://demo.ckan.org/api/3/action/dashboard_activity_list')
+    request = urllib2.Request('https://demo.ckan.org/api/3/action/dashboard_activity_list')
     request.add_header('Authorization', 'XXX')
     response_dict = json.loads(urllib2.urlopen(request, '{}').read())
 
@@ -374,9 +374,9 @@ Uploading a new version of a resource file
 You can use the ``upload`` parameter of the
 :py:func:`~ckan.logic.action.update.resource_update` function to upload a
 new version of a resource file. This requires a ``multipart/form-data``
-request, with httpie you can do this using the ``@file.csv``::
+request, with curl you can do this using the ``@file.csv``::
 
-    http --json POST http://demo.ckan.org/api/3/action/resource_update id=<resource id> upload=@updated_file.csv Authorization:<api key>
+    curl -X POST  -H "Content-Type: multipart/form-data"  -H "Authorization: XXXX"  -F "id=<resource_id>" -F "upload=@updated_file.csv" https://demo.ckan.org/api/3/action/resource_update  
 
 
 .. _api-reference:
diff --git a/doc/contributing/documentation.rst b/doc/contributing/documentation.rst
index b45f45e77c6..bbe6cf16db0 100644
--- a/doc/contributing/documentation.rst
+++ b/doc/contributing/documentation.rst
@@ -492,7 +492,7 @@ is a nice way to include a list of related links::
        :doc:`The DataStore extension <datastore>`
          A CKAN extension for storing data.
 
-       CKAN's `demo site <http://demo.ckan.org/>`_
+       CKAN's `demo site <https://demo.ckan.org/>`_
          A demo site running the latest CKAN beta version.
 
 Seealso boxes are particularly useful when two pages are related, but don't
diff --git a/doc/maintaining/linked-data-and-rdf.rst b/doc/maintaining/linked-data-and-rdf.rst
index 80676ee9f25..8f52fedef5b 100644
--- a/doc/maintaining/linked-data-and-rdf.rst
+++ b/doc/maintaining/linked-data-and-rdf.rst
@@ -9,10 +9,10 @@ https://github.com/ckan/ckanext-dcat
 These features include the RDF serializations of CKAN datasets based on `DCAT`_, that used to be generated
 using templates hosted on the main CKAN repo, eg:
 
-* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.xml
-* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.ttl
-* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.n3
-* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.jsonld
+* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.xml
+* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.ttl
+* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.n3
+* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.jsonld
 
 ckanext-dcat offers many more `features <https://github.com/ckan/ckanext-dcat#overview>`_,
 including catalog-wide endpoints and harvesters to import RDF data into CKAN. Please check
diff --git a/doc/maintaining/stats.rst b/doc/maintaining/stats.rst
index fee5f4cfa60..15b76192025 100644
--- a/doc/maintaining/stats.rst
+++ b/doc/maintaining/stats.rst
@@ -42,4 +42,4 @@ Viewing the Statistics
 ======================
 
 To view the statistics reported by the stats extension, visit the ``/stats``
-page, for example: http://demo.ckan.org/stats
+page, for example: https://demo.ckan.org/stats
diff --git a/doc/user-guide.rst b/doc/user-guide.rst
index 51ea26d4c43..cb40061736e 100644
--- a/doc/user-guide.rst
+++ b/doc/user-guide.rst
@@ -140,7 +140,7 @@ Adding a new dataset
 
     You may need to be a member of an organization in order to add and edit
     datsets. See the section :ref:`creating_an_organization` below. On
-    http://demo.ckan.org, you can add a dataset without being in an organization,
+    https://demo.ckan.org, you can add a dataset without being in an organization,
     but dataset features relating to authorization and organizations will not be
     available.
 

From 37fb5bc9be2d81dcfa2a2ae42bc532b7a2c61e02 Mon Sep 17 00:00:00 2001
From: Ross Jones <ross@mailbolt.com>
Date: Fri, 8 Jun 2018 08:29:27 +0100
Subject: [PATCH 24/24] Fix a missing http->https change

---
 doc/api/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/api/index.rst b/doc/api/index.rst
index 8f42754441d..1fb3e117089 100644
--- a/doc/api/index.rst
+++ b/doc/api/index.rst
@@ -79,7 +79,7 @@ of all the datasets in the ``data-explorer`` group on demo.ckan.org, install
 curl and then call the ``group_list`` API function by running this command
 in a terminal::
 
-    curl http://demo.ckan.org/api/3/action/group_list
+    curl https://demo.ckan.org/api/3/action/group_list
 
 The response from CKAN will look like this::