Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
94 lines (88 sloc) 2.19 KB

How to export all datasets from data.govt.nz into a CSV

Summary

Using the ckanapi-exporter tool you can extract all dataset metadata into a single CSV file.

Requirements

Steps

  1. Install requirements
  2. Create a columns.json file (sets out some preset data propoerties to extract from data.govt.nz API, you can customise if required). See below for file contents.
  3. Run the below command on your terminal
ckanapi-exporter --url 'https://catalogue.data.govt.nz' --columns columns.json > datasets.csv

columns.json

{
 "Title": {
     "pattern": "^title$"
 },
 "Agency": {
     "pattern": ["^organization$", "^title$"]
 },
 "URL": {
     "pattern": "^url$"
 },
 "CatalogueCreated": {
     "pattern": "^metadata_created$",
     "max_length": 10
 },
 "CatalogueLastUpdated": {
     "pattern": "^metadata_modified$",
     "max_length": 10
 },
 "DatasetCreated": {
     "pattern": "^issued$",
     "max_length": 10
 },
 "DatasetLastUpdated": {
     "pattern": "^modified$",
     "max_length": 10
 },
 "FrequencyOfUpdate": {
     "pattern": "^frequency_of_update$"
 },
 "Rights": {
     "pattern": "^license_title$"
 },
 "FormatsAvailable": {
     "pattern": ["^resources$", "^format$"],
     "case_sensitive": true,
     "deduplicate": true
 },
 "Description": {
     "pattern": "^notes$"
 },
 "Tags": {
   "pattern": ["^tags$", "^display_name$"]
 },
 "Groups": {
     "pattern": ["^groups$", "^display_name$"]
 },
 "AgencyContact": {
     "pattern": "^author$"
 },
 "AgencyContactEmail":{
     "pattern": "^author_email$"
 },
 "AgencyContactPhone":{
     "pattern": "^author_phone$"
 },
 "DatasetContact": {
     "pattern": "^maintainer$"
 },
 "DatasetContactEmail": {
     "pattern": "^maintainer_email$"
 },
 "DatasetContactPhone": {
     "pattern": "^maintainer_phone$"
 },
 "PermanentIdentifier":{
     "pattern": "^id$"
 },
 "SourceIdentifier": {
     "pattern": "^source_identifier$"
 }
}