cli: improve wayback cli to take optional port, threads and working d…

…ir arguments

switch to waitress as default WSGI server instead of wsgiref

.gitattributes

@@ -1,4 +1,7 @@
 *.arc  -text
 *.warc -text
+*.idx  -text
+*.idxj -text
 *.cdx  -text
+*.cdxj -text
 *.gz   -text

pywb/apps/cli.py

@@ -0,0 +1,32 @@
+#=================================================================
+def wayback(args=None):
+    from argparse import ArgumentParser, RawTextHelpFormatter
+
+    parser = ArgumentParser('pywb Wayback Web Archive Replay')
+    parser.add_argument('-p', '--port', type=int, default=8080)
+    parser.add_argument('-t', '--threads', type=int, default=4)
+
+    help_dir='Specify root archive dir (default is current working directory)'
+    parser.add_argument('-d', '--directory', help=help_dir)
+
+    r = parser.parse_args(args)
+    if r.directory:  #pragma: no cover
+        import os
+        os.chdir(r.directory)
+
+    # Load App
+    from pywb.apps.wayback import application
+
+    try:
+        from waitress import serve
+        serve(application, port=r.port, threads=r.threads)
+    except ImportError:  # pragma: no cover
+        # Shouldn't ever happen as installing waitress, but just in case..
+        from pywb.framework.wsgi_wrappers import start_wsgi_server
+        start_wsgi_server(application, 'Wayback', default_port=r.port)
+
+
+#=================================================================
+if __name__ == "__main__":
+    wayback()
+

pywb/apps/wayback.py

@@ -1,14 +1,8 @@
-from pywb.framework.wsgi_wrappers import init_app, start_wsgi_server
+from pywb.framework.wsgi_wrappers import init_app
 from pywb.webapp.pywb_init import create_wb_router
 
+
 #=================================================================
 # init pywb app
 #=================================================================
 application = init_app(create_wb_router, load_yaml=True)
-
-
-def main():  # pragma: no cover
-    start_wsgi_server(application, 'Wayback')
-
-if __name__ == "__main__":
-    main()

pywb/manager/manager.py

@@ -55,6 +55,10 @@ def _set_coll_dirs(self, coll_name):
 
     def list_colls(self):
         print('Collections:')
+        if not os.path.isdir(self.colls_dir):
+            msg = ('"Collections" directory not found. ' +
+                   'To create a new collection, run:\n\n{0} init <name>')
+            raise IOError(msg.format(sys.argv[0]))
         for d in os.listdir(self.colls_dir):
             if os.path.isdir(os.path.join(self.colls_dir, d)):
                 print('- ' + d)
@@ -87,8 +91,9 @@ def add_collection(self):
 
     def _assert_coll_exists(self):
         if not os.path.isdir(self.curr_coll_dir):
-            raise IOError('Collection {0} does not exist'.
-                          format(self.coll_name))
+            msg = ('Collection {0} does not exist. ' +
+                   'To create a new collection, run\n\n{1} init {0}')
+            raise IOError(msg.format(self.coll_name, sys.argv[0]))
 
     def add_warcs(self, warcs):
         if not os.path.isdir(self.archive_dir):
@@ -466,7 +471,6 @@ def main_wrap_exc():  #pragma: no cover
     try:
         main()
     except Exception as e:
-        raise
         print('Error: ' + str(e))
         sys.exit(2)
 

setup.py

@@ -75,6 +75,7 @@ def run_tests(self):
         'surt',
         'pyyaml',
         'youtube_dl',
+        'waitress',
         'watchdog'
        ],
     tests_require=[
@@ -88,7 +89,7 @@ def run_tests(self):
     test_suite='',
     entry_points="""
         [console_scripts]
-        wayback = pywb.apps.wayback:main
+        wayback = pywb.apps.cli:wayback
         cdx-server = pywb.apps.cdx_server:main
         cdx-indexer = pywb.warc.cdxindexer:main
         live-rewrite-server = pywb.apps.live_rewrite_server:main

tests/test_auto_colls.py

@@ -70,6 +70,13 @@ def _check_dirs(self, base, dirlist):
     def _get_sample_warc(self, name):
         return os.path.join(get_test_dir(), 'warcs', name)
 
+    @patch('waitress.serve', lambda *args, **kwargs: None)
+    def test_run_cli(self):
+        """ test new wayback cli interface
+        """
+        from pywb.apps.cli import wayback
+        wayback([])
+
     def test_create_first_coll(self):
         """ Test first collection creation, with all required dirs
         """
@@ -561,6 +568,10 @@ def test_err_missing_dirs(self):
 
         shutil.rmtree(colls)
 
+        # No Collections to list
+        with raises(IOError):
+            main(['list'])
+
         # No Collections
         self._create_app()
         resp = self.testapp.get('/test/', status=404)