implement core metadata CRUD workflow, update CLI architecture

.gitignore

@@ -100,3 +100,6 @@ ENV/
 
 # debian artifacts
 .pybuild
+
+# SQLite databases
+*.db

.travis.yml

@@ -5,7 +5,9 @@ python:
 
 env:
   global:
-    - WDR_DEBUG=True
+    - WDR_LOGGING_LOGLEVEL=ERROR
+    - WDR_LOGGING_LOGFILE=/tmp/woudc-data-registry.log
+    - WDR_DB_DEBUG=False
     - WDR_DB_TYPE=postgresql
     - WDR_DB_HOST=localhost
     - WDR_DB_PORT=5432
@@ -29,6 +31,7 @@ services:
 
 install:
   - pip install -r requirements-dev.txt
+  - pip install -r requirements-pg.txt
   - pip install coveralls
   - python setup.py build
   - python setup.py install

Makefile

@@ -18,7 +18,7 @@
 # those files. Users are asked to read the 3rd Party Licenses
 # referenced with those assets.
 #
-# Copyright (c) 2017 Government of Canada
+# Copyright (c) 2019 Government of Canada
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -88,22 +88,25 @@ dropdb:
 flake8:
 	flake8 woudc_data_registry
 
+init:
+	woudc-data-registry admin init -d $(DATA)
+
 package:
 	python setup.py sdist bdist_wheel
 
 setup:
-	woudc-data-registry manage setup
-	woudc-data-registry search create-index
+	woudc-data-registry admin setup
+	woudc-data-registry admin search create-index
 
 setup_data:
 	mkdir -p data
 	curl -o data/wmo-countries.json https://www.wmo.int/cpdb/data/membersandterritories.json
 
 teardown:
-	woudc-data-registry manage teardown
-	woudc-data-registry search delete-index
+	woudc-data-registry admin teardown
+	woudc-data-registry admin search delete-index
 
 test:
 	python setup.py test
 
-.PHONY: clean coverage createdb dropdb flake8 help package setup setup_data teardown test
+.PHONY: clean coverage createdb dropdb flake8 help init package setup setup_data teardown test

README.md

@@ -5,23 +5,22 @@
 
 ## Overview
 
-WOUDC Data Registry is a platform that manages Ozone and Ultraviolet
-Radiation data in support of the [World Ozone and Ultraviolet Radiation Data
+WOUDC Data Registry is a platform that manages ozone and ultraviolet
+radiation data in support of the [World Ozone and Ultraviolet Radiation Data
 Centre (WOUDC)](https://woudc.org), one of six World Data Centres as part of
 the [Global Atmosphere Watch](http://www.wmo.int/gaw) programme of the
-[WMO](http://www.wmo.int).
-
+[WMO](https://wmo.int).
 
 ## Installation
 
 ### Requirements
-- [Python](https://www.python.org) 3 and above
+- [Python](https://python.org) 3 and above
 - [virtualenv](https://virtualenv.pypa.io/)
 - [Elasticsearch](https://www.elastic.co/products/elasticsearch) (5.5.0 and above)
 
 ### Dependencies
 Dependencies are listed in [requirements.txt](requirements.txt). Dependencies
-are automatically installed during woudc-data-registry installation.
+are automatically installed during installation.
 
 ### Installing woudc-data-registry
 
@@ -36,6 +35,9 @@ git clone https://github.com/woudc/woudc-data-registry.git
 cd woudc-data-registry
 python setup.py build
 python setup.py install
+# for PostgreSQL backends
+pip install -r requirements-pg.txt
+
 
 # set system environment variables
 cp default.env foo.env
@@ -49,27 +51,27 @@ make ENV=foo.env createdb
 make ENV=foo.env dropdb
 
 # initialize model (database tables)
-woudc-data-registry manage setup
+woudc-data-registry admin setup
 
 # initialize search engine
-woudc-data-registry search create-index
+woudc-data-registry admin search create-index
 
 # load core metadata
 
 # fetch WMO country list
 mkdir data
 curl -o data/wmo-countries.json https://www.wmo.int/cpdb/data/membersandterritories.json
-woudc-data-registry manage init -d data/
+woudc-data-registry admin init -d data/
 
 # cleanups
 
 # re-initialize model (database tables)
-woudc-data-registry manage teardown
-woudc-data-registry manage setup
+woudc-data-registry admin teardown
+woudc-data-registry admin setup
 
 # re-initialize search engine
-woudc-data-registry search delete-index
-woudc-data-registry search create-index
+woudc-data-registry admin search delete-index
+woudc-data-registry admin search create-index
 
 # drop database
 make ENV=foo.env dropdb
@@ -78,6 +80,33 @@ make ENV=foo.env dropdb
 
 ### Running woudc-data-registry
 
+TIP: autocompletion can be made available in some shells via:
+
+```bash
+eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"
+```
+
+#### Core Metadata Management
+
+```bash
+# list all contributors
+woudc-data-registry contributor list
+
+# show a single contributor details
+woudc-data-registry contributor show MSC
+
+# add a contributor
+woudc-data-registry contributor add -id foo -n "Contributor name" -c Canada -w IV -u https://example.org -e you@example.org -f foouser -g -75,45
+
+# update a contributor
+woudc-data-registry contributor update -id foo -n "New Contributor name"
+
+# delete a contributor
+woudc-data-registry contributor delete foo
+```
+
+#### Data Processing
+
 ```bash
 # ingest directory of files (walks directory recursively)
 woudc-data-registry data ingest -d /path/to/dir
@@ -86,11 +115,10 @@ woudc-data-registry data ingest -d /path/to/dir
 woudc-data-registry data ingest -f foo.dat
 
 # verify directory of files (walks directory recursively)
-woudc-data-registry data ingest -d /path/to/dir --verify
+woudc-data-registry data verify -d /path/to/dir
 
 # verify single file
-woudc-data-registry data ingest -f foo.dat --verify
-
+woudc-data-registry data verify -f foo.dat
 ```
 
 ### Running Tests

debian/control

@@ -9,7 +9,7 @@ Vcs-Git: https://github.com/woudc/woudc-data-registry.git
 
 Package: woudc-data-registry
 Architecture: all
-Depends: elasticsearch (>=5.5.0), postgresql, python3-click, python-elasticsearch, python3-psycopg2, python3-requests, python3-six, python3-sqlalchemy
+Depends: elasticsearch (>=5.5.0), postgresql, python3-click, python-elasticsearch, python3-psycopg2, python3-requests, python3-sqlalchemy
 Homepage: https://woudc.org
 Description: WOUDC Data Registry is a platform that manages Ozone and
  Ultraviolet Radiation data in support of the World Ozone and Ultraviolet

debian/copyright

@@ -2,7 +2,7 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Source: https://github.com/woudc/woudc-data-registry
 
 Files: *
-Copyright: Copyright (c) 2017 Government of Canada
+Copyright: Copyright (c) 2019 Government of Canada
 License: Expat
  Permission is hereby granted, free of charge, to any person
  obtaining a copy of this software and associated documentation 

default.env

@@ -1,13 +1,28 @@
-export WDR_DEBUG=False
+export WDR_LOGGING_LOGLEVEL=DEBUG
+export WDR_LOGGING_LOGFILE=stdout
+
+# database configuration
+export WDR_DB_DEBUG=True
+
+# PostgreSQL
 export WDR_DB_TYPE=postgresql
 export WDR_DB_HOST=localhost
 export WDR_DB_PORT=5432
 export WDR_DB_NAME=woudc-data-registry
 export WDR_DB_USERNAME=postgres
 export WDR_DB_PASSWORD=postgres
+
+# SQLite
+export WDR_DB_TYPE=sqlite
+export WDR_DB_NAME=test.db
+
+
 export WDR_SEARCH_TYPE=elasticsearch
 export WDR_SEARCH_URL=http://localhost:9200/
 export WDR_WAF_BASEURL=https://woudc.org/archive/
 export WDR_WAF_BASEDIR=/tmp
 
 export PGPASSWORD=$WDR_DB_PASSWORD
+
+# enable shell autocompletion
+eval "$(_WOUDC_DATA_REGISTRY_COMPLETE=source woudc-data-registry)"

requirements-pg.txt

@@ -0,0 +1 @@
+psycopg2

requirements.txt

@@ -1,6 +1,4 @@
 click
 elasticsearch
-psycopg2
 requests
-six
 sqlalchemy

woudc_data_registry/__init__.py

@@ -46,21 +46,26 @@
 import click
 
 from woudc_data_registry.controller import data
-from woudc_data_registry.epicentre import contributor, dataset
-from woudc_data_registry.models import manage
-from woudc_data_registry.search import search
+from woudc_data_registry import config, epicentre
+from woudc_data_registry.epicentre.contributor import contributor
+from woudc_data_registry.epicentre.station import station
+from woudc_data_registry.models import admin
+from woudc_data_registry.log import setup_logger
 
 __version__ = '0.1.dev0'
 
+setup_logger(config.WDR_LOGGING_LOGLEVEL, config.WDR_LOGGING_LOGFILE)
+
 
 @click.group()
 @click.version_option(version=__version__)
 def cli():
     pass
 
 
-cli.add_command(manage)
+cli.add_command(admin)
 cli.add_command(data)
-cli.add_command(search)
 cli.add_command(contributor)
-cli.add_command(dataset)
+cli.add_command(station)
+cli.add_command(epicentre.dataset)
+cli.add_command(epicentre.project)

woudc_data_registry/config.py

@@ -18,7 +18,7 @@
 # those files. Users are asked to read the 3rd Party Licenses
 # referenced with those assets.
 #
-# Copyright (c) 2017 Government of Canada
+# Copyright (c) 2019 Government of Canada
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -50,29 +50,39 @@
 
 LOGGER = logging.getLogger(__name__)
 
+WDR_LOGGING_LOGLEVEL = os.getenv('WDR_LOGGING_LOGLEVEL', 'ERROR')
+WDR_LOGGING_LOGFILE = os.getenv('WDR_LOGGING_LOGFILE', None)
 
-WDR_DEBUG = str2bool(os.getenv('WDR_DEBUG', False))
-
-WDR_DB_TYPE = os.getenv('WDR_DB_TYPE', 'postgresql')
-WDR_DB_HOST = os.getenv('WDR_DB_HOST', 'localhost')
+WDR_DB_DEBUG = str2bool(os.getenv('WDR_DB_DEBUG', False))
+WDR_DB_TYPE = os.getenv('WDR_DB_TYPE', None)
+WDR_DB_HOST = os.getenv('WDR_DB_HOST', None)
 WDR_DB_PORT = int(os.getenv('WDR_DB_PORT', 5432))
 WDR_DB_USERNAME = os.getenv('WDR_DB_USERNAME', None)
 WDR_DB_PASSWORD = os.getenv('WDR_DB_PASSWORD', None)
-WDR_DB_NAME = os.getenv('WDR_DB_NAME', 'woudc-data-registry')
+WDR_DB_NAME = os.getenv('WDR_DB_NAME', None)
 WDR_SEARCH_TYPE = os.getenv('WDR_SEARCH_TYPE', 'elasticsearch')
-WDR_SEARCH_URL = os.getenv('WDR_SEARCH_URL', 'elasticsearch')
+WDR_SEARCH_URL = os.getenv('WDR_SEARCH_URL', None)
 WDR_WAF_BASEDIR = os.getenv('WDR_WAF_BASEDIR', None)
 WDR_WAF_BASEURL = os.getenv('WDR_WAF_BASEURL', 'https://woudc.org/archive')
 
-if None in [WDR_DB_USERNAME, WDR_DB_PASSWORD, WDR_SEARCH_TYPE,
-            WDR_SEARCH_URL, WDR_WAF_BASEDIR, WDR_WAF_BASEURL]:
-    msg = 'System environment variables are not set!'
+if WDR_DB_TYPE is None:
+    msg = 'WDR_DB_TYPE is not set!'
     LOGGER.error(msg)
     raise EnvironmentError(msg)
 
 if WDR_DB_TYPE == 'sqlite':
+    if WDR_DB_NAME is None:
+        msg = 'WDR_DB_NAME e is not set!'
+        LOGGER.error(msg)
+        raise EnvironmentError(msg)
     WDR_DATABASE_URL = '{}:///{}'.format(WDR_DB_TYPE, WDR_DB_NAME)
 else:
+    if None in [WDR_DB_USERNAME, WDR_DB_PASSWORD, WDR_SEARCH_TYPE,
+                WDR_SEARCH_URL, WDR_WAF_BASEDIR, WDR_WAF_BASEURL]:
+        msg = 'System environment variables are not set!'
+        LOGGER.error(msg)
+        raise EnvironmentError(msg)
+
     WDR_DATABASE_URL = '{}://{}:{}@{}:{}/{}'.format(WDR_DB_TYPE,
                                                     WDR_DB_USERNAME,
                                                     WDR_DB_PASSWORD,