Merge pull request #65 from OpenGeoMetadata/ogp

This PR adds support for basic OGP -> GeoBlacklight metadata conversion

lib/geo_combine.rb

@@ -66,6 +66,7 @@ def to_html
 require 'geo_combine/iso19139'
 require 'geo_combine/esri_open_data'
 require 'geo_combine/ckan_metadata'
+require 'geo_combine/ogp'
 
 # Require gem files
 require 'geo_combine/version'

lib/geo_combine/formatting.rb

@@ -25,5 +25,9 @@ def remove_lines(text)
     def sanitize_and_remove_lines(text)
       remove_lines(sanitize(text))
     end
+
+    def sluggify(text)
+      URI.encode(text.tr('.', '-').tr('_', '-').downcase)
+    end
   end
 end

lib/geo_combine/ogp.rb

@@ -0,0 +1,155 @@
+module GeoCombine
+  # Data model for ESRI's open data portal metadata
+  class OGP
+    include GeoCombine::Formatting
+    attr_reader :metadata
+
+    ##
+    # Initializes an OGP object for parsing
+    # @param [String] metadata a valid serialized JSON string from an ESRI Open
+    # Data portal
+    def initialize(metadata)
+      @metadata = JSON.parse(metadata)
+    end
+
+    ##
+    # Creates and returns a Geoblacklight schema object from this metadata
+    # @return [GeoCombine::Geoblacklight]
+    def to_geoblacklight
+      GeoCombine::Geoblacklight.new(geoblacklight_terms.to_json)
+    end
+
+    ##
+    # Builds a Geoblacklight Schema type hash from Esri Open Data portal
+    # metadata
+    # @return [Hash]
+    def geoblacklight_terms
+      {
+        # Required fields
+        dc_identifier_s: URI.encode(metadata['LayerId']),
+        layer_slug_s: sluggify(metadata['LayerId']),
+        dc_title_s: metadata['LayerDisplayName'],
+        solr_geom: envelope,
+        dct_provenance_s: metadata['Institution'],
+        dc_rights_s: metadata['Access'],
+        geoblacklight_version: '1.0',
+
+        # Recommended fields
+        dc_description_s: metadata['Abstract'],
+        layer_geom_type_s: ogp_geom,
+        dct_references_s: references,
+        layer_id_s: "#{metadata['WorkspaceName']}:#{metadata['Name']}",
+
+        # Optional
+        dct_temporal_sm: [metadata['ContentDate']],
+        dc_format_s: ogp_formats,
+        # dct_issued_dt
+        # dc_language_s
+        dct_spatial_sm: placenames,
+        solr_year_i: year,
+        dc_publisher_s: metadata['Publisher'],
+        dc_subject_sm: subjects,
+        dc_type_s: 'Dataset'
+      }.delete_if { |_k, v| v.nil? }
+    end
+
+    def date
+      begin
+        DateTime.rfc3339(metadata['ContentDate'])
+      rescue
+        nil
+      end
+    end
+
+    def year
+      date.year unless date.nil?
+    end
+
+    ##
+    # Convert "Paper Map" to Raster, assumes all OGP "Paper Maps" have WMS
+    def ogp_geom
+      case metadata['DataType']
+      when 'Paper Map'
+        'Raster'
+      else
+        metadata['DataType']
+      end
+    end
+
+    ##
+    # OGP doesn't ship format types, so we just try and be clever here.
+    def ogp_formats
+      case metadata['DataType']
+      when 'Paper Map', 'Raster'
+        return 'GeoTIFF'
+      when 'Polygon', 'Point', 'Line'
+        return 'Shapefile'
+      else
+        ''
+      end
+    end
+
+    ##
+    # Converts references to json
+    # @return [String]
+    def references
+      references_hash.to_json
+    end
+
+    ##
+    # Builds a Solr Envelope using CQL syntax
+    # @return [String]
+    def envelope
+      "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
+    end
+
+    def subjects
+      fgdc.metadata.xpath('//themekey').map { |k| k.text }
+    end
+
+    def placenames
+      fgdc.metadata.xpath('//placekey').map { |k| k.text }
+    end
+
+    def fgdc
+      GeoCombine::Fgdc.new(metadata['FgdcText'])
+    end
+
+    private
+
+    ##
+    # Builds references used for dct_references
+    # @return [Hash]
+    def references_hash
+      {
+        'http://www.opengis.net/def/serviceType/ogc/wfs' => location['wfs'],
+        'http://www.opengis.net/def/serviceType/ogc/wms' => location['wms'],
+        'http://schema.org/DownloadAction' => location['download']
+        # Handle null, "", and [""]
+      }.map { |k, v| { k => ([] << v).flatten.first } if v }
+        .flatten.compact.reduce({}, :merge)
+    end
+
+    ##
+    # OGP "Location" field parsed
+    def location
+      JSON.parse(metadata['Location'])
+    end
+
+    def north
+      @metadata['MaxY']
+    end
+
+    def south
+      @metadata['MinY']
+    end
+
+    def east
+      @metadata['MaxX']
+    end
+
+    def west
+      @metadata['MinX']
+    end
+  end
+end

spec/fixtures/docs/ogp_harvard_raster.json

@@ -0,0 +1,32 @@
+{
+  "LayerId": "HARVARD.SDE2.G1059_W57_1654_PF_SH1",
+  "Name": "G1059_W57_1654_PF_SH1",
+  "CollectionId": "initial collection",
+  "ExternalLayerId": "1188",
+  "Institution": "Harvard",
+  "Access": "Public",
+  "DataType": "Paper Map",
+  "Availability": "online",
+  "LayerDisplayName": "World Map, 1654 (Raster Image)",
+  "Publisher": "Harvard Map Collection, Harvard Library",
+  "Originator": "Harvard Map Collection, Harvard Library",
+  "OriginatorSort": "Harvard Map Collection, Harvard Library",
+  "ThemeKeywords": "Human settlements Landforms imageryBaseMapsEarthCover Transportation Bodies water Maps and towns of Villages Cities",
+  "PlaceKeywords": "regions Europe Africa Hemisphere South Earth Western Asia Northern America Eastern North Southern Australia Arctic",
+  "PlaceKeywordsSynonyms": "regions Europe Africa Hemisphere South Earth Western Asia Northern America Eastern North Southern Australia Arctic",
+  "GeoReferenced": true,
+  "Abstract": "This layer is a georeferenced raster image of the historic paper map entitled: Nova totius terrarum orbis geographica ac hydrographica tabula, autore N.I. Piscator, from the Frederik de Wit's untitled Dutch Sea Atlas. It was published by F. de Wit in 1654. Scale [ca. 1:100,000,000]. Map in Latin.\n\nThe image inside the map neatline is georeferenced to the surface of the earth and fit to the World Geodetic System 1984 (WGS 84) coordinate system. All map collar and inset information is also available as part of the raster image, including any inset maps, profiles, statistical tables, directories, text, illustrations, index maps, legends, or other information associated with the principal map. \n\nThis map shows features such as drainage, cities and other human settlements, territorial boundaries, shoreline features, and more. Relief shown pictorially. \n\nThis layer is part of a selection of digitally scanned and georeferenced historic maps from the Harvard Map Collection. These maps typically portray both natural and manmade features. The selection represents a range of originators, ground condition dates, scales, and map purposes.",
+  "Location": "{\"wms\": [\"http://pelham.lib.harvard.edu:8090/geoserver/wms\"],\"wcs\": \"http://pelham.lib.harvard.edu:8090/geoserver/wcs\",\"download\": \"http://pelham.lib.harvard.edu:8080/HGL/HGLOpenDelivery\"}",
+  "MaxY": 90,
+  "MinY": -90,
+  "MaxX": 180,
+  "MinX": -180,
+  "CenterX": 0,
+  "CenterY": 0,
+  "HalfWidth": 180,
+  "HalfHeight": 90,
+  "Area": 64800,
+  "WorkspaceName": "cite",
+  "ContentDate": "1654-01-01T01:01:01Z",
+  "FgdcText": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE metadata SYSTEM \"http://www.fgdc.gov/metadata/fgdc-std-001-1998.dtd\"><metadata><idinfo><citation><citeinfo><origin>Harvard Map Collection, Harvard Library</origin><origin>Visscher, Claes Jansz., 1586 or 1587-1652.</origin><origin>Wit, Frederik de.</origin><pubdate>201404</pubdate><title>World Map, 1654 (Raster Image)</title><geoform>map</geoform><pubinfo><pubplace>Cambridge, Massachusetts</pubplace><publish>Harvard Map Collection, Harvard Library</publish></pubinfo><onlink>http://hgl.harvard.edu:8080/HGL/hgl.jsp?action=VCollandamp;VCollName=G1059_W57_1654_PF_SH1</onlink></citeinfo></citation><descript><abstract>This layer is a georeferenced raster image of the historic paper map entitled: Nova totius terrarum orbis geographica ac hydrographica tabula, autore N.I. Piscator, from the Frederik de Wit's untitled Dutch Sea Atlas. It was published by F. de Wit in 1654. Scale [ca. 1:100,000,000]. Map in Latin.The image inside the map neatline is georeferenced to the surface of the earth and fit to the World Geodetic System 1984 (WGS 84) coordinate system. All map collar and inset information is also available as part of the raster image, including any inset maps, profiles, statistical tables, directories, text, illustrations, index maps, legends, or other information associated with the principal map. This map shows features such as drainage, cities and other human settlements, territorial boundaries, shoreline features, and more. Relief shown pictorially. This layer is part of a selection of digitally scanned and georeferenced historic maps from the Harvard Map Collection. These maps typically portray both natural and manmade features. The selection represents a range of originators, ground condition dates, scales, and map purposes.</abstract><purpose>Historic paper maps can provide an excellent view of the changes that have occurred in the cultural and physical landscape.  The wide range of information provided on these maps make them useful in the study of historic geography, and urban and rural land use change.  As this map has been georeferenced, it can be used in a GIS as a source or background layer in conjunction with other GIS data.</purpose></descript><timeperd><timeinfo><sngdate><caldate>1654</caldate></sngdate></timeinfo><current>source map publication date</current></timeperd><status><progress>Complete</progress><update>None planned</update></status><spdom><bounding><westbc>-180</westbc><eastbc>180</eastbc><northbc>90</northbc><southbc>-90</southbc></bounding></spdom><keywords><theme><themekt>LCSH</themekt><themekey>Maps</themekey><themekey>Human settlements</themekey><themekey>Cities and towns</themekey><themekey>Villages</themekey><themekey>Bodies of water</themekey><themekey>Landforms</themekey><themekey>Transportation</themekey></theme><theme><themekt>ISO 19115 Topic Category</themekt><themekey>imageryBaseMapsEarthCover</themekey></theme><place><placekt>LCSH</placekt><placekey>Earth</placekey><placekey>Northern Hemisphere</placekey><placekey>Southern Hemisphere</placekey><placekey>Eastern Hemisphere</placekey><placekey>Western Hemisphere</placekey><placekey>Africa</placekey><placekey>Asia</placekey><placekey>Australia</placekey><placekey>Europe</placekey><placekey>North America</placekey><placekey>South America</placekey><placekey>Arctic regions</placekey></place></keywords><accconst>None</accconst><useconst>For educational, non-commercial use only.</useconst><ptcontac><cntinfo><cntorgp><cntorg>Harvard Map Collection, Harvard Library</cntorg></cntorgp><cntpos>Harvard Geospatial Library</cntpos><cntaddr><addrtype>mailing and physical address</addrtype><address>Harvard Map Collection</address><address>Pusey Library</address><address>Harvard University</address><city>Cambridge</city><state>MA</state><postal>02138</postal><country>USA</country></cntaddr><cntvoice>617-495-2417</cntvoice><cntfax>617-496-0440</cntfax><cntemail>hgl_ref@hulmail.harvard.edu</cntemail><hours>Monday - Friday, 9:00 am - 4:00 pm EST-USA</hours></cntinfo></ptcontac><native>ESRI ArcGIS 10.1</native></idinfo><dataqual><attracc><attraccr>The georeferenced raster is a faithfully reproduced digital image of the original source map. Some differences may be detected between the source graphic used and the raster image due to the RGB values assigned that particular color. The intent is to recreate those colors as near as possible.</attraccr></attracc><complete>Data completeness for raster digital image files reflect content of the source graphic. Features may have been eliminated or generalized on the source graphic due to scale and legibility constraints</complete><posacc><horizpa><horizpar>The horizontal positional accuracy of a raster image is approximately the same as the accuracy of the published source map.  The lack of a greater accuracy is largely the result of the inaccuracies with the original measurements and possible distortions in the original paper map document. There may also be errors introduced during the digitizing and georeferencing process. In most cases, however, errors in the raster image are small compared with sources of error in the original map graphic.  The RMS error for this map is 7.40181 Decimal Degrees.  This value describes how consistent the transformation is between the different control points (links).  The RMS error is only an assessment of the accuracy of the transformation.</horizpar></horizpa></posacc><lineage><srcinfo><srccite><citeinfo><origin>Visscher, Claes Jansz., 1586 or 1587-1652.</origin><origin>Wit, Frederik de.</origin><pubdate>1654</pubdate><title>Nova totius terrarum orbis geographica ac hydrographica tabula</title><geoform>map</geoform><pubinfo><pubplace>Amsterdam</pubplace><publish>F. de Wit</publish></pubinfo><othercit>1 map : hand col. ; 29 x 43 cm., on sheet 49 x 57 cm.</othercit><lworkcit><citeinfo><origin>Wit, Frederik de.</origin><origin>Visscher, Claes Jansz., 1586 or 1587-1652.</origin><origin>Blaeu, Willem Janszoon, 1571-1638.</origin><origin>Jacobsz, Anthonie. 1606?-1650.</origin><origin>Jacobsz, Theunis, 1632?-1679.</origin><pubdate>1654</pubdate><title>[Dutch sea atlas].</title><geoform>atlas</geoform><pubinfo><pubplace>Amsterdam</pubplace><publish>F. de Wit</publish></pubinfo><othercit>1 atlas : 15 col. maps ; 50 x 61 cm.</othercit></citeinfo></lworkcit></citeinfo></srccite><srcscale>100000000</srcscale><typesrc>paper</typesrc><srctime><timeinfo><sngdate><caldate>1654</caldate></sngdate></timeinfo><srccurr>publication date</srccurr></srctime><srccitea>Paper Map</srccitea><srccontr>Source map for raster image</srccontr></srcinfo><procstep><procdesc>Production of this raster image began with the scanning of the paper map on a high-resolution scanner (Colortrac Smart LF Gx+T42 scanner and SmartWorksPro version 1.1.1.5 scanning software) at 400dpi.  Resulting images were rotated and cropped as needed using Adobe Photoshop CS5 Extended version 12.0. Images were saved as lossless JPEG2000 files.</procdesc><srcused>Paper Map</srcused><procdate>2012</procdate><srcprod>JP2 Map Image</srcprod><proccont><cntinfo><cntorgp><cntorg>Harvard Geospatial Library</cntorg></cntorgp><cntpos>GIS Technician</cntpos><cntaddr><addrtype>mailing and physical address</addrtype><address>Harvard University Information Technology</address><address>Library Technology Services</address><address>90 Mount Auburn Street</address><city>Cambridge</city><state>MA</state><postal>02138</postal><country>USA</country></cntaddr><cntvoice>617-495-2417</cntvoice><cntfax>617-496-0440</cntfax><cntemail>hgl_ref@hulmail.harvard.edu</cntemail><hours>Monday - Friday, 9:00 am - 4:00 pm EST-USA</hours></cntinfo></proccont></procstep><procstep><procdesc>Using ArcMap 10.1 software, the digital JPEG2000 image was georeferenced to common points located on digital vector shapefiles. For this image, the following vector shapefile was used as a base map for reference: ESRI Data and Maps 2010: World Country Boundaries (1:10,000,000). The base map data is displayed in the World Geodetic System 1984 (WGS 84) coordinate system, and the image was subsequently georeferenced to the same projection. A world file (.j2w) was automatically generated and saved in association with the digital image. See 'Horizontal Accuracy' for the RMS error of this image. Please note that the projection will need to be defined by the user in order to display the image with other projected data and the world file will need to be stored in the same root directory as the image.</procdesc><srcused>JP2 Map Image</srcused><procdate>2012</procdate><srcprod>Georeferenced Raster Data</srcprod><proccont><cntinfo><cntorgp><cntorg>Harvard Geospatial Library</cntorg></cntorgp><cntpos>GIS Technician</cntpos><cntaddr><addrtype>mailing and physical address</addrtype><address>Harvard University Information Technology</address><address>Library Technology Services</address><address>90 Mount Auburn Street</address><city>Cambridge</city><state>MA</state><postal>02138</postal><country>USA</country></cntaddr><cntvoice>617-495-2417</cntvoice><cntfax>617-496-0440</cntfax><cntemail>hgl_ref@hulmail.harvard.edu</cntemail><hours>Monday - Friday, 9:00 am - 4:00 pm EST-USA</hours></cntinfo></proccont></procstep></lineage></dataqual><spdoinfo><direct>Raster</direct><rastinfo><rasttype>Pixel</rasttype><rowcount>7606</rowcount><colcount>8936</colcount><vrtcount>1</vrtcount></rastinfo></spdoinfo><spref><horizsys><geograph><latres>0.000001</latres><longres>0.000001</longres><geogunit>Decimal degrees</geogunit></geograph><geodetic><horizdn>D_WGS_1984</horizdn><ellips>WGS_1984</ellips><semiaxis>6378137.000000</semiaxis><denflat>298.257224</denflat></geodetic></horizsys></spref><eainfo><overview><eaover>The indexes reference a color palette of RGB values from 0 through 255, representing the color value from the original paper sheet map. The colors on that sheet can represent relief, drainage, vegetation, populated places, cultural features, coastal hydrography, transportation features (roads, railroads, tracks and trails), spot elevations and boundaries. The colors are sometimes explained in a legend that is incorporated into the map inset or collar.</eaover><eadetcit>Not applicable.</eadetcit></overview></eainfo><distinfo><distrib><cntinfo><cntorgp><cntorg>Harvard University Information Technology</cntorg></cntorgp><cntpos>Harvard Geospatial Library</cntpos><cntaddr><addrtype>mailing and physical address</addrtype><address>Library Technology Services</address><address>90 Mount Auburn Street</address><city>Cambridge</city><state>MA</state><postal>02138</postal><country>USA</country></cntaddr><cntvoice>617-495-2417</cntvoice><cntfax>617-496-0440</cntfax><cntemail>hgl_ref@hulmail.harvard.edu</cntemail><hours>Monday - Friday, 9:00 am - 4:00 pm EST-USA</hours></cntinfo></distrib><resdesc>Downloadable Data</resdesc><distliab>Although this data set has been developed by Harvard University, no warranty expressed or implied is made by the University as to the accuracy of the data and related materials. The act of distribution shall not constitute any such warranty, and no responsibility is assumed by the University in the use of this data, or related materials.</distliab><stdorder><digform><digtinfo><formname>JPEG2000</formname><filedec>ZIP</filedec></digtinfo><digtopt><onlinopt><computer><networka><networkr>http://hgl.harvard.edu/</networkr></networka></computer></onlinopt></digtopt></digform><fees>None</fees></stdorder></distinfo><metainfo><metd>20140422</metd><metc><cntinfo><cntorgp><cntorg>Harvard Geospatial Library</cntorg></cntorgp><cntpos>Geospatial Metadata Librarian</cntpos><cntaddr><addrtype>mailing and physical address</addrtype><address>Harvard University Information Technology</address><address>Library Technology Services</address><address>90 Mount Auburn Street</address><city>Cambridge</city><state>MA</state><postal>02138</postal><country>USA</country></cntaddr><cntvoice>617-495-2417</cntvoice><cntfax>617-496-0440</cntfax><cntemail>hgl_ref@hulmail.harvard.edu</cntemail><hours>Monday - Friday, 9:00 am - 4:00 pm EST-USA</hours></cntinfo></metc><metstdn>FGDC Content Standards for Digital Geospatial Metadata</metstdn><metstdv>FGDC-STD-001-1998</metstdv><mettc>local time</mettc></metainfo></metadata>"
+}