Merge pull request #55 from open-contracting/better-cli

cli: Add many options, convert to click

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased] 
 
+### Added
+
+- Added many options to CLI: convert, output-dir, schema-version, delete, exclude-file
+
 ### Changed
 
 - Cache all requests in the tests https://github.com/OpenDataServices/lib-cove/pull/59

README.rst

@@ -12,6 +12,19 @@ Call ``libcoveocds`` and pass the filename of some JSON data.
 
    libcoveocds tests/fixtures/common_checks/basic_1.json
 
+It will produce JSON data of the results to standard output. You can pipe this straight into a file to work with.
+
+You can also pass ``--schema-version 1.X`` to force it to check against a certain version of the schema.
+
+In some modes, it will also leave directory of data behind. The following options apply to this mode:
+
+* Pass ``--convert`` to get it to produce spreadsheets of the data.
+* Pass ``--output-dir output`` to specify a directory name (default is a name based on the filename).
+* Pass ``--delete`` to delete the output directory if it already exists (default is to error)
+* Pass ``--exclude`` to avoid copying the original file into the output directory (default is to copy)
+
+(If none of these are specified, it will not leave any files behind)
+
 Code for use by external users
 ------------------------------
 

libcoveocds/cli/__main__.py

@@ -1,32 +1,78 @@
-import argparse
 import json
+import os
 import shutil
+import sys
 import tempfile
 
+import click
+
 import libcoveocds.api
+from libcoveocds.config import LibCoveOCDSConfig
+
+
+@click.command()
+@click.argument('filename')
+@click.option('-c', '--convert', is_flag=True, help='Convert data from nested (json) to flat format (spreadsheet)')
+@click.option('-o', '--output-dir', default=None,
+              help="Directory where the output is created, defaults to the name of the file")
+@click.option('-s', '--schema-version', default=None, help="Version of the schema to validate the data, eg '1.0'")
+@click.option('-d', '--delete', is_flag=True, help='Delete existing directory if it exits')
+@click.option('-e', '--exclude-file', is_flag=True, help='Do not include the file in the output directory')
+def process(filename, output_dir, convert, schema_version, delete, exclude_file):
 
+    if schema_version:
+        lib_cove_ocds_config = LibCoveOCDSConfig()
+        version_choices = lib_cove_ocds_config.config.get('schema_version_choices')
+        if schema_version not in version_choices:
+            print(
+                "Value for schema version option is not valid. Accepted values: {}".format(
+                    ", ".join(version_choices)
+                )
+            )
+            sys.exit(1)
 
-def main():
-    parser = argparse.ArgumentParser(description='Lib Cove OCDS CLI')
-    parser.add_argument("filename")
+    # Do we have output on disk? We only do in certain modes
+    has_disk_output = output_dir or convert or delete or exclude_file
+    if has_disk_output:
+        if not output_dir:
+            output_dir = filename.split('/')[-1].split('.')[0]
 
-    args = parser.parse_args()
+        if os.path.exists(output_dir):
+            if delete:
+                shutil.rmtree(output_dir)
+            else:
+                print('Directory {} already exists'.format(output_dir))
+                sys.exit(1)
+        os.makedirs(output_dir)
+
+        if not exclude_file:
+            shutil.copy2(filename, output_dir)
+    else:
+        # If not, just put in /tmp and delete after
+        output_dir = tempfile.mkdtemp(
+            prefix='lib-cove-ocds-cli-',
+            dir=tempfile.gettempdir(),
+        )
 
-    cove_temp_folder = tempfile.mkdtemp(prefix='lib-cove-ocds-cli-', dir=tempfile.gettempdir())
     try:
         result = libcoveocds.api.ocds_json_output(
-            cove_temp_folder,
-            args.filename,
-            None,
-            convert=False,
+            output_dir,
+            filename,
+            schema_version,
+            convert=convert,
             cache_schema=True,
             file_type='json'
         )
     finally:
-        shutil.rmtree(cove_temp_folder)
+        if not has_disk_output:
+            shutil.rmtree(output_dir)
+
+    if has_disk_output:
+        with open(os.path.join(output_dir, 'results.json'), "w") as fp:
+            fp.write(json.dumps(result, indent=2))
 
-    print(json.dumps(result, indent=4))
+    print(json.dumps(result, indent=2))
 
 
 if __name__ == '__main__':
-    main()
+    process()

setup.py

@@ -21,6 +21,7 @@
         'Django',
         'json-merge-patch',
         'requests',
+        'click',
     ],
     extras_require={
         'test': [

tests/test_cli.py

@@ -0,0 +1,111 @@
+import json
+import os
+import shutil
+import tempfile
+
+from click.testing import CliRunner
+
+from libcoveocds.cli.__main__ import process
+
+
+def test_basic():
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    assert result.exit_code == 0
+    data = json.loads(result.output)
+    assert "1.1" == data.get('version_used')
+
+
+def test_old_schema():
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        '-s 1.0 ' + os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    print(result.output)
+    assert result.exit_code == 0
+    data = json.loads(result.output)
+    assert "1.0" == data.get('version_used')
+
+
+def test_set_output_dir():
+    output_dir = tempfile.mkdtemp(
+        prefix='lib-cove-ocds-tests-',
+        dir=tempfile.gettempdir(),
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        ' -o ' + output_dir + ' ' + os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    # This will fail because tempfile.mkdtemp already will make the directory, and so it already exists
+    assert result.exit_code == 1
+    assert result.output.startswith('Directory ')
+    assert result.output.endswith('already exists\n')
+    shutil.rmtree(output_dir)
+
+
+def test_set_output_dir_and_delete():
+    output_dir = tempfile.mkdtemp(
+        prefix='lib-cove-ocds-tests-',
+        dir=tempfile.gettempdir(),
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        ' -d -o ' + output_dir + ' ' + os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    assert result.exit_code == 0
+    data = json.loads(result.output)
+    assert "1.1" == data.get('version_used')
+    # Should have results file and original file and nothing else
+    expected_files = ['basic_1.json', 'results.json']
+    assert sorted(os.listdir(output_dir)) == sorted(expected_files)
+    shutil.rmtree(output_dir)
+
+
+def test_set_output_dir_and_delete_and_exclude():
+    output_dir = tempfile.mkdtemp(
+        prefix='lib-cove-ocds-tests-',
+        dir=tempfile.gettempdir(),
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        ' -d -e -o ' + output_dir + ' ' + os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    assert result.exit_code == 0
+    data = json.loads(result.output)
+    assert "1.1" == data.get('version_used')
+    # Should have results file only
+    expected_files = ['results.json']
+    assert sorted(os.listdir(output_dir)) == sorted(expected_files)
+    shutil.rmtree(output_dir)
+
+
+def test_set_output_dir_and_convert():
+    output_dir = tempfile.mkdtemp(
+        prefix='lib-cove-ocds-tests-',
+        dir=tempfile.gettempdir(),
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        process,
+        ' -c -d -o ' + output_dir + ' ' + os.path.join('tests', 'fixtures', 'common_checks', 'basic_1.json')
+    )
+    assert result.exit_code == 0
+    data = json.loads(result.output)
+    assert "1.1" == data.get('version_used')
+    # Should have results file and original file and the converted files
+    expected_files = ['basic_1.json', 'flattened', 'flattened.ods', 'flattened.xlsx', 'results.json']
+    assert sorted(os.listdir(output_dir)) == sorted(expected_files)
+    # Flattened should be a directory of csv's.
+    # We aren't going to check names fully
+    # That would leave this test brittle if flatten-tools naming scheme changed,
+    # so we will just check extension
+    for filename in os.listdir(os.path.join(output_dir, 'flattened')):
+        assert filename.endswith('.csv')
+    shutil.rmtree(output_dir)