Support weaker SSL/TLS connections for a broader compatibility with o…

…utdated web servers

wpull's `--no-strong-crypto` allows for SSLv2 and SSLv3 connections (if the OpenSSL library is built accordingly) and compression.
The OpenSSL configuration file overrides the defaults that are shipped e.g. with Debian Buster, which only accept TLSv1.2+ and decently secure cipher suites.

INSTALL.pipeline

@@ -77,6 +77,12 @@ As user archivebot, in the SECOND tmux session:
     export REDIS_URL=redis://127.0.0.1:16379/0
     export FINISHED_WARCS_DIR=$HOME/warcs4fos
 
+If you run the pipeline on a system with a modern OpenSSL version
+(e.g. Debian Buster and later), which comes with a more secure default
+configuration, additionally set the OPENSSL_CONF environment variable:
+
+    export OPENSSL_CONF=/home/archivebot/ArchiveBot/ops/openssl-less-secure.cnf
+
 Now, think up a name for this new ArchiveBot pipeline.  It will 
 appear on the publicly available pipeline status dashboard. It will 
 go in the command you enter next:

ops/openssl-less-secure.cnf

@@ -0,0 +1,2 @@
+MinProtocol = None
+CipherString = DEFAULT

pipeline/archivebot/seesaw/wpull.py

@@ -24,6 +24,7 @@ def make_args(item, default_user_agent, wpull_exe, youtube_dl_exe, finished_warc
         '--html-parser', 'libxml2-lxml',
         '--save-cookies', '%(cookie_jar)s' % item,
         '--no-check-certificate',
+        '--no-strong-crypto',
         '--delete-after',
         '--no-robots',
         '--page-requisites',

pipeline/pipeline.py

@@ -75,6 +75,7 @@
 REDIS_URL = env['REDIS_URL']
 LOG_CHANNEL = shared_config.log_channel()
 PIPELINE_CHANNEL = shared_config.pipeline_channel()
+OPENSSL_CONF = env.get('OPENSSL_CONF')
 
 # ------------------------------------------------------------------------------
 # CONTROL CONNECTION
@@ -132,6 +133,14 @@ def __contains__(self, item):
     monitor_disk=WPULL_MONITOR_DISK,
     monitor_memory=WPULL_MONITOR_MEMORY,
 )
+wpull_env = {
+    'ITEM_IDENT': ItemInterpolation('%(ident)s'),
+    'LOG_KEY': ItemInterpolation('%(log_key)s'),
+    'REDIS_URL': REDIS_URL,
+    'PATH': os.environ['PATH'],
+}
+if OPENSSL_CONF:
+    wpull_env['OPENSSL_CONF'] = OPENSSL_CONF
 
 pipeline = Pipeline(
     CheckIP(),
@@ -147,12 +156,7 @@ def __contains__(self, item):
     Wpull(
         wpull_args,
         accept_on_exit_code=AcceptAny(),
-        env={
-            'ITEM_IDENT': ItemInterpolation('%(ident)s'),
-            'LOG_KEY': ItemInterpolation('%(log_key)s'),
-            'REDIS_URL': REDIS_URL,
-            'PATH': os.environ['PATH']
-        }
+        env=wpull_env,
     ),
     RelabelIfAborted(control),
     CompressLogIfFailed(),