add do-not-upload option

This option is useful coupled with the local-save option and will
generate an upload tar without trying to actually upload anything.

Signed-off-by: David Brown <dmlb2000@gmail.com>

README.md

@@ -1,6 +1,6 @@
 # Pacifica CLI Uploader
 
-[![Build Status](https://travis-ci.org/pacifica/pacifica-cli-uploader.svg?branch=master)](https://travis-ci.org/pacifica/pacifica-cli)
+[![Build Status](https://travis-ci.org/pacifica/pacifica-cli.svg?branch=master)](https://travis-ci.org/pacifica/pacifica-cli)
 [![Build Status](https://ci.appveyor.com/api/projects/status/0ddinx1bdfroptf7?svg=true)](https://ci.appveyor.com/project/dmlb2000/pacifica-cli)
 [![Code Climate](https://codeclimate.com/github/pacifica/pacifica-cli/badges/gpa.svg)](https://codeclimate.com/github/pacifica/pacifica-cli)
 [![Issue Count](https://codeclimate.com/github/pacifica/pacifica-cli/badges/issue_count.svg)](https://codeclimate.com/github/pacifica/pacifica-cli)

appveyor.yml

@@ -41,7 +41,7 @@ before_test:
     $env:INGEST_CONFIG = "$PWD/travis/ingest/config.cfg";
     $env:INGEST_CPCONFIG = "$PWD/travis/ingest/server.conf";
     Start-Process C:\pacifica\Scripts\pacifica-ingest.exe;
-    Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.ingest.tasks worker --loglevel=info -P eventlet";
+    Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.ingest.tasks worker --loglevel=info -P solo -c 1" -RedirectStandardOutput celery-output.log -RedirectStandardError celery-error.log;
     $MD_VERSION = `pip show pacifica-metadata | grep Version: | awk '{ print $2 }';
     Invoke-WebRequest https://github.com/pacifica/pacifica-metadata/archive/v${MD_VERSION}.zip -OutFile pacifica-metadata.zip;
     Expand-Archive pacifica-metadata.zip -DestinationPath C:\pacifica-metadata;
@@ -55,6 +55,11 @@ before_test:
     Start-Process C:\pacifica\Scripts\pacifica-policy.exe -RedirectStandardError policy-error.log -RedirectStandardOutput policy-output.log;
     sleep 3;
     Invoke-WebRequest http://127.0.0.1:8181/status/users/search/dmlb2001/simple -TimeoutSec 1800;
+    Invoke-WebRequest 'http://127.0.0.1:8066/get_state?job_id=1234';
+    Invoke-WebRequest 'http://127.0.0.1:8051/getid?range=42&mode=test_mode';
+    Invoke-WebRequest http://127.0.0.1:8080/1234;
+    Get-Content celery-output.log; Get-Content celery-error.log;
+    echo "Done!";
 
 install:
 - ps: >
@@ -66,7 +71,7 @@ install:
     C:\pacifica\Scripts\activate.ps1;
     python -m pip install pip setuptools wheel --upgrade;
     pip install -r requirements-dev.txt;
-    pip install celery[redis] eventlet;
+    pip install celery[redis] 'redis<3.0';
 
 build: off
 
@@ -103,10 +108,14 @@ test_script:
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --dry-run --logon dmlb2001 --proposal-regex 'expired closed and end';
     Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } -Body '{ "network_id":"appveyor" }' http://localhost:8121/users?_id=10;
     Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } -Body '{ "network_id":"someoneelse" }' http://localhost:8121/users?_id=11;
+    coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli --verbose debug upload --nowait README.md;
+    sleep 10;
+    Get-Content "..\celery-output.log"; Get-Content "..\celery-error.log";
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload travis;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --tar-in-tar README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar README.md;
+    coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar --do-not-upload README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-retry retry.tar;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --nowait README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pytest -xv;

pacifica/cli/__main__.py

@@ -82,6 +82,10 @@ def main():
         '--dry-run', default=False, action='store_true', dest='dry_run',
         help='Don\'t upload, stop after query engine.', required=False
     )
+    upload_parser.add_argument(
+        '--do-not-upload', default=False, action='store_true', dest='do_not_upload',
+        help='Don\'t upload, works well with local save option.', required=False
+    )
     upload_parser.add_argument(
         '--interactive', default=False, action='store_true', dest='interactive',
         help='Interact with the query engine.', required=False

pacifica/cli/upload.py

@@ -177,9 +177,31 @@ def perform_upload(md_update, args, content_length, tar_size):
     rfd, wfd = setup_chain_thread(
         (rfd, wfd), (args.localsave,), save_local, wthreads, args.localsave)
     setup_bundler(wfd, md_update, args, wthreads)
-    up_obj = Uploader(auth=md_update.get_auth())
     LOGGER.debug('Starting with rfd (%s) and wfd (%s) and %s threads %s',
                  rfd, wfd, len(wthreads), content_length)
+    if args.do_not_upload:
+        jobid, up_obj = fake_uploader(rfd, content_length)
+    else:
+        jobid, up_obj = invoke_uploader(md_update, rfd, content_length)
+    for wthread in wthreads:
+        wthread.join()
+    LOGGER.debug('Threads completd')
+    rfd.close()
+    return jobid, up_obj
+
+
+def fake_uploader(rfd, content_length):
+    """Fake the upload by reading all the content then returning."""
+    read_data = 0
+    while read_data < content_length:
+        buf = rfd.read(1024)
+        read_data += len(buf)
+    return -1, None
+
+
+def invoke_uploader(md_update, rfd, content_length):
+    """Invoke the uploader code to actually upload."""
+    up_obj = Uploader(auth=md_update.get_auth())
 
     # pylint: disable=too-few-public-methods
     class FakeFileObj(object):
@@ -197,16 +219,12 @@ def read(self, size=-1):
 
     jobid = up_obj.upload(FakeFileObj(rfd, content_length),
                           content_length=content_length)
-    for wthread in wthreads:
-        wthread.join()
-    LOGGER.debug('Threads completd')
-    rfd.close()
     return jobid, up_obj
 
 
 def wait_for_upload(args, jobid, up_obj):
     """Wait (or not) for the jobid to complete the ingest process."""
-    if not args.wait:
+    if not args.wait or args.do_not_upload:
         print('Not Waiting Job ID ({})'.format(jobid))
         return 0
     print('Waiting job to complete ({}).'.format(jobid))

travis/unit-tests.sh

@@ -56,6 +56,7 @@ $COV_RUN -a -m pacifica.cli upload README.md
 $COV_RUN -a -m pacifica.cli upload travis
 $COV_RUN -a -m pacifica.cli upload --tar-in-tar README.md
 $COV_RUN -a -m pacifica.cli upload --local-save retry.tar README.md
+$COV_RUN -a -m pacifica.cli upload --local-save retry.tar --do-not-upload README.md
 $COV_RUN -a -m pacifica.cli upload --local-retry retry.tar
 $COV_RUN -a -m pacifica.cli upload --nowait README.md