Merge branch 'master' into remove-vdm

ckan · Oct 31, 2019 · f7bc190 · f7bc190
2 parents 5cbd6f7 + 986e1ca
commit f7bc190
Show file tree

Hide file tree

Showing 18 changed files with 777 additions and 856 deletions.
diff --git a/ckan/cli/datastore.py b/ckan/cli/datastore.py
@@ -11,7 +11,7 @@
 
 import ckanext.datastore as datastore_module
 from ckanext.datastore.backend.postgres import identifier
-from ckanext.datastore.view import DUMP_FORMATS, dump_to
+from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to
 
 log = logging.getLogger(__name__)
 

diff --git a/ckan/config/resource_formats.json b/ckan/config/resource_formats.json
@@ -8,70 +8,84 @@
    " * Mimetype - canonical mimetype for the format. It must be unique to this resource format. It should be listed here: https://www.iana.org/assignments/media-types/media-types.xhtml or here: http://hg.python.org/cpython/file/2.7/Lib/mimetypes.py#l403",
    " * List of alternative representations - these are other names that the user might type when they mean this format, or alternative mime-types or any other identifier. (They must be unique to this resource format.)"
   ],
-  ["PPTX", "Powerpoint OOXML Presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation", []],
-  ["EXE", "Windows Executable Program", "application/x-msdownload", []],
+  ["ArcGIS Map Preview", "ArcGIS Map Preview", "ArcGIS Map Preview", ["arcgis map preview"]],
+  ["ArcGIS Map Service", "ArcGIS Map Service", "ArcGIS Map Service", ["arcgis map service"]],
+  ["ArcGIS Online Map", "ArcGIS Online Map", "ArcGIS Online Map", ["web map application"]],
+  ["Atom Feed", "Atom Feed", "application/atom+xml", []],
+  ["BIN", "Binary Data", "application/octet-stream", ["bin"]],
+  ["BMP", "Bitmap Image File", "image/x-ms-bmp", []],
+  ["CSV", "Comma Separated Values File", "text/csv", ["text/comma-separated-values"]],
+  ["DCR", "Adobe Shockwave format", "application/x-director", []],
+  ["dBase", "dBase Database", "application/x-dbf", ["dbf"]],
   ["DOC", "Word Document", "application/msword", []],
-  ["KML", "KML File", "application/vnd.google-earth.kml+xml", []],
-  ["XLS", "Excel Document", "application/vnd.ms-excel", ["Excel", "application/msexcel", "application/x-msexcel", "application/x-ms-excel", "application/x-excel", "application/x-dos_ms_excel", "application/xls", "application/x-xls"]],
-  ["WCS", "Web Coverage Service", "wcs", []],
+  ["DOCM", "Word OOXML - Macro Enabled", "application/vnd.ms-word.document.macroEnabled.12", []],
+  ["DOCX", "Word OOXML Document", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", []],
+  ["DOTM", "Word OOXML Template - Macro Enabled", "application/vnd.ms-word.template.macroEnabled.12", []],
+  ["DOTX", "Word OOXML - Template", "application/vnd.openxmlformats-officedocument.wordprocessingml.template", []],
+  ["Esri REST", "Esri Rest API Endpoint", "Esri REST", ["arcgis_rest"]],
+  ["EXE", "Windows Executable Program", "application/x-msdownload", []],
+  ["E00", " ARC/INFO interchange file format", "application/x-e00", []],
+  ["GeoJSON", "Geographic JavaScript Object Notation", "application/geo+json", ["geojson"]],
+  ["GIF", "GIF Image File", "image/gif", []],
+  ["GTFS", "General Transit Feed Specification", null, []],
+  ["GZ", "Gzip File", "application/gzip", ["application/x-gzip"]],
+  ["HTML", "Web Page", "text/html", ["htm", "http://purl.org/net/mediatypes/text/html"]],
+  ["ICS", "iCalendar", "text/calendar", ["ifb", "iCal"]],
+  ["JPEG", "JPG Image File", "image/jpeg", ["jpeg", "jpg"]],
   ["JS", "JavaScript", "application/x-javascript", []],
+  ["JSON", "JavaScript Object Notation", "application/json", []],
+  ["KML", "KML File", "application/vnd.google-earth.kml+xml", []],
+  ["KMZ", "KMZ File", "application/vnd.google-earth.kmz+xml", ["application/vnd.google-earth.kmz"]],
   ["MDB", "Access Database", "application/x-msaccess", []],
+  ["MOP", "MOPAC Input format", "chemical/x-mopac-input", []],
+  ["MrSID", "MrSID", "image/x-mrsid", []],
+  ["MXD", "ESRI ArcGIS project file", "application/x-mxd", []],
   ["NetCDF", "NetCDF File", "application/netcdf", []],
-  ["ArcGIS Map Service", "ArcGIS Map Service", "ArcGIS Map Service", ["arcgis map service"]],
-  ["TSV", "Tab Separated Values File", "text/tab-separated-values", ["text/tsv"]],
-  ["WFS", "Web Feature Service", null, []],
-  ["WMTS", "Web Map Tile Service", null, []],
-  ["ArcGIS Online Map", "ArcGIS Online Map", "ArcGIS Online Map", ["web map application"]],
-  ["Perl", "Perl Script", "text/x-perl", []],
-  ["KMZ", "KMZ File", "application/vnd.google-earth.kmz+xml", ["application/vnd.google-earth.kmz"]],
-  ["OWL", "Web Ontology Language", "application/owl+xml", []],
   ["N3", "N3 Triples", "application/x-n3", []],
-  ["ZIP", "Zip File", "application/zip", ["zip", "http://purl.org/NET/mediatypes/application/zip"]],
-  ["GZ", "Gzip File", "application/gzip", ["application/x-gzip"]],
-  ["QGIS", "QGIS File", "application/x-qgis", []],
-  ["ODS", "OpenDocument Spreadsheet", "application/vnd.oasis.opendocument.spreadsheet", []],
-  ["ODT", "OpenDocument Text", "application/vnd.oasis.opendocument.text", []],
-  ["JSON", "JavaScript Object Notation", "application/json", []],
-  ["BMP", "Bitmap Image File", "image/x-ms-bmp", []],
-  ["HTML", "Web Page", "text/html", ["htm", "http://purl.org/net/mediatypes/text/html"]],
-  ["RAR", "RAR Compressed File", "application/rar", []],
-  ["TIFF", "TIFF Image File", "image/tiff", []],
   ["ODB", "OpenDocument Database", "application/vnd.oasis.opendocument.database", []],
-  ["TXT", "Text File", "text/plain", []],
-  ["DCR", "Adobe Shockwave format", "application/x-director", []],
+  ["ODC", "OpenDocument Chart", "application/vnd.oasis.opendocument.chart", []],
   ["ODF", "OpenDocument Math Formula", "application/vnd.oasis.opendocument.formula", []],
   ["ODG", "OpenDocument Image", "application/vnd.oasis.opendocument.graphics", []],
-  ["XML", "XML File", "application/xml", ["text/xml", "http://purl.org/net/mediatypes/application/xml"]],
-  ["XLSX", "Excel OOXML Spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", []],
-  ["DOCX", "Word OOXML Document", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", []],
-  ["BIN", "Binary Data", "application/octet-stream", ["bin"]],
-  ["XSLT", "Extensible Stylesheet Language Transformations", "application/xslt+xml", []],
-  ["WMS", "Web Mapping Service", "WMS", ["wms"]],
-  ["SVG", "SVG vector image", "image/svg+xml", ["svg"]],
-  ["PPT", "Powerpoint Presentation", "application/vnd.ms-powerpoint", []],
+  ["OWL", "Web Ontology Language", "application/owl+xml", []],
   ["ODP", "OpenDocument Presentation", "application/vnd.oasis.opendocument.presentation", []],
-  ["JPEG", "JPG Image File", "image/jpeg", ["jpeg", "jpg"]],
-  ["SPARQL", "SPARQL end-point", "application/sparql-results+xml", []],
-  ["GIF", "GIF Image File", "image/gif", []],
-  ["RDF", "RDF", "application/rdf+xml", ["rdf/xml"]],
-  ["E00", " ARC/INFO interchange file format", "application/x-e00", []],
+  ["ODS", "OpenDocument Spreadsheet", "application/vnd.oasis.opendocument.spreadsheet", []],
+  ["ODT", "OpenDocument Text", "application/vnd.oasis.opendocument.text", []],
   ["PDF", "PDF File", "application/pdf", []],
-  ["CSV", "Comma Separated Values File", "text/csv", ["text/comma-separated-values"]],
-  ["ODC", "OpenDocument Chart", "application/vnd.oasis.opendocument.chart", []],
-  ["Atom Feed", "Atom Feed", "application/atom+xml", []],
-  ["MrSID", "MrSID", "image/x-mrsid", []],
-  ["ArcGIS Map Preview", "ArcGIS Map Preview", "ArcGIS Map Preview", ["arcgis map preview"]],
-  ["XYZ", "XYZ Chemical File", "chemical/x-xyz", []],
-  ["MOP", "MOPAC Input format", "chemical/x-mopac-input", []],
-  ["Esri REST", "Esri Rest API Endpoint", "Esri REST", ["arcgis_rest"]],
-  ["dBase", "dBase Database", "application/x-dbf", ["dbf"]],
-  ["MXD", "ESRI ArcGIS project file", "application/x-mxd", []],
-  ["TAR", "TAR Compressed File", "application/x-tar", []],
+  ["Perl", "Perl Script", "text/x-perl", []],
   ["PNG", "PNG Image File", "image/png", []],
+  ["POTM", "PowerPoint Open XML - Macro-Enabled Template", "application/vnd.ms-powerpoint.template.macroEnabled.12", []],
+  ["POTX", "PowerPoint OOXML Presentation - Template", "application/vnd.openxmlformats-officedocument.presentationml.template", []],
+  ["PPAM", "PowerPoint Open XML - Macro-Enabled Add-In", "application/vnd.ms-powerpoint.addin.macroEnabled.12", []],
+  ["PPT", "Powerpoint Presentation", "application/vnd.ms-powerpoint", []],
+  ["PPTM", "PowerPoint Open XML - Macro-Enabled", "application/vnd.ms-powerpoint.presentation.macroEnabled.12", []],
+  ["PPTX", "Powerpoint OOXML Presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation", []],
+  ["PPSM", "PowerPoint Open XML - Macro-Enabled Slide Show", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12", []],
+  ["PPSX", "PowerPoint Open XML - Slide Show", "application/vnd.openxmlformats-officedocument.presentationml.slideshow", []],
+  ["QGIS", "QGIS File", "application/x-qgis", []],
+  ["RAR", "RAR Compressed File", "application/rar", []],
+  ["RDF", "RDF", "application/rdf+xml", ["rdf/xml"]],
   ["RSS", "RSS feed", "application/rss+xml", []],
-  ["GeoJSON", "Geographic JavaScript Object Notation", "application/geo+json", ["geojson"]],
   ["SHP", "Shapefile", null, ["esri shapefile"]],
+  ["SPARQL", "SPARQL end-point", "application/sparql-results+xml", []],
+  ["SVG", "SVG vector image", "image/svg+xml", ["svg"]],
+  ["TAR", "TAR Compressed File", "application/x-tar", []],
+  ["TIFF", "TIFF Image File", "image/tiff", []],
   ["TORRENT", "Torrent", "application/x-bittorrent", ["bittorrent"]],
-  ["ICS", "iCalendar", "text/calendar", ["ifb", "iCal"]],
-  ["GTFS", "General Transit Feed Specification", null, []]
+  ["TSV", "Tab Separated Values File", "text/tab-separated-values", ["text/tsv"]],
+  ["TXT", "Text File", "text/plain", []],
+  ["WCS", "Web Coverage Service", "wcs", []],
+  ["WFS", "Web Feature Service", null, []],
+  ["WMS", "Web Mapping Service", "WMS", ["wms"]],
+  ["WMTS", "Web Map Tile Service", null, []],
+  ["XLAM", "Excel OOXML Spreadsheet - Macro-Enabled Add-In", "application/vnd.ms-excel.addin.macroEnabled.12", []],
+  ["XLS", "Excel Document", "application/vnd.ms-excel", ["Excel", "application/msexcel", "application/x-msexcel", "application/x-ms-excel", "application/x-excel", "application/x-dos_ms_excel", "application/xls", "application/x-xls"]],
+  ["XLSB", "Excel OOXML Spreadsheet - Binary Workbook", "application/vnd.ms-excel.sheet.binary.macroEnabled.12", []],
+  ["XLSM", "Excel OOXML Spreadsheet - Macro-Enabled", "application/vnd.ms-excel.sheet.macroEnabled.12", []],
+  ["XLSX", "Excel OOXML Spreadsheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", []],
+  ["XLTM", "Excel OOXML Spreadsheet - Macro-Enabled Template", "application/vnd.ms-excel.template.macroEnabled.12", []],
+  ["XLTX", "Excel OOXML Spreadsheet - Template", "application/vnd.openxmlformats-officedocument.spreadsheetml.template", []],
+  ["XML", "XML File", "application/xml", ["text/xml", "http://purl.org/net/mediatypes/application/xml"]],
+  ["XSLT", "Extensible Stylesheet Language Transformations", "application/xslt+xml", []],
+  ["XYZ", "XYZ Chemical File", "chemical/x-xyz", []],
+  ["ZIP", "Zip File", "application/zip", ["zip", "http://purl.org/NET/mediatypes/application/zip"]]
 ]
diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py
@@ -46,7 +46,11 @@ def resource_dict_save(res_dict, context):
             continue
         if key in fields:
             if isinstance(getattr(obj, key), datetime.datetime):
-                if getattr(obj, key).isoformat() == value:
+                if isinstance(value, string_types):
+                    db_value = getattr(obj, key).isoformat()
+                else:
+                    db_value = getattr(obj, key)
+                if  db_value == value:
                     continue
                 if key == 'last_modified' and not new:
                     obj.url_changed = True

diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py
@@ -462,11 +462,9 @@ class TestPep8(object):
         'ckanext/datastore/logic/action.py',
         'ckanext/datastore/tests/test_create.py',
         'ckanext/datastore/tests/test_search.py',
-        'ckanext/datastore/tests/test_upsert.py',
         'ckanext/example_idatasetform/plugin.py',
         'ckanext/example_itemplatehelpers/plugin.py',
         'ckanext/multilingual/plugin.py',
-        'ckanext/resourceproxy/plugin.py',
         'ckanext/stats/controller.py',
         'ckanext/stats/stats.py',
         'ckanext/stats/tests/test_stats_plugin.py',

diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
@@ -584,7 +584,6 @@ def find_unprefixed_string_literals(filename):
     u'ckanext/datapusher/tests/test_action.py',
     u'ckanext/datapusher/tests/test_default_views.py',
     u'ckanext/datapusher/tests/test_interfaces.py',
-    u'ckanext/datastore/controller.py',
     u'ckanext/datastore/helpers.py',
     u'ckanext/datastore/backend/postgres.py',
     u'ckanext/datastore/interfaces.py',
@@ -613,7 +612,6 @@ def find_unprefixed_string_literals(filename):
     u'ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py',
     u'ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py',
     u'ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py',
-    u'ckanext/example_iconfigurer/controller.py',
     u'ckanext/example_iconfigurer/plugin.py',
     u'ckanext/example_iconfigurer/plugin_v1.py',
     u'ckanext/example_iconfigurer/plugin_v2.py',

diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
@@ -365,6 +365,10 @@ def _where_clauses(data_dict, fields_types):
         if field not in fields_types:
             continue
         field_array_type = _is_array_type(fields_types[field])
+        # "%" needs to be escaped as "%%" in any query going to
+        # connection.execute, otherwise it will think the "%" is for
+        # substituting a bind parameter
+        field = field.replace('%', '%%')
         if isinstance(value, list) and not field_array_type:
             clause_str = (u'"{0}" in ({1})'.format(field,
                           ','.join(['%s'] * len(value))))
@@ -1148,8 +1152,9 @@ def upsert_data(context, data_dict):
                                     WHERE ({primary_key}) = ({primary_value}));
                 '''.format(
                     res_id=data_dict['resource_id'],
-                    columns=u', '.join([u'"{0}"'.format(field)
-                                        for field in used_field_names]),
+                    columns=u', '.join([
+                        u'"{0}"'.format(field.replace('%', '%%'))
+                        for field in used_field_names]),
                     values=u', '.join(['%s::nested'
                                        if field['type'] == 'nested' else '%s'
                                        for field in used_fields]),

diff --git a/ckanext/datastore/view.py → ckanext/datastore/blueprint.py b/ckanext/datastore/view.py → ckanext/datastore/blueprint.py
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
@@ -9,7 +9,7 @@
     click_config_option,
 )
 from ckanext.datastore.backend.postgres import identifier
-from ckanext.datastore.view import DUMP_FORMATS, dump_to
+from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to
 
 import click