Docs test merge pr

.circleci/config.yml

@@ -74,7 +74,7 @@ jobs:
           name: Run all linters but shellcheck
           command: |
             fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "/opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint docs-lint flake8 html-lint typelint yamllint"
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "/opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint flake8 html-lint typelint yamllint"
 
       - run:
           name: Run shellcheck
@@ -277,7 +277,7 @@ jobs:
             if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-deb-tests
 
-  docs-linkcheck:
+  docs-lint-linkcheck:
     docker:
       - image: circleci/python:3.5
     steps:
@@ -297,11 +297,23 @@ workflows:
   version: 2
   securedrop_ci:
     jobs:
-      - lint
+      - docs-lint-linkcheck:
+          filters:
+            branches:
+              only:
+                - master
+                - /docs-.*/
+      - lint:
+          filters:
+            branches:
+              ignore:
+                - master
+                - /docs-.*/
       - app-tests:
           filters:
             branches:
               ignore:
+                - master
                 - /docs-.*/
                 - /i18n-.*/
                 - /update-builder-.*/
@@ -311,6 +323,7 @@ workflows:
           filters:
             branches:
               ignore:
+                - master
                 - /docs-.*/
                 - /i18n-.*/
                 - /update-builder-.*/
@@ -320,23 +333,35 @@ workflows:
           filters:
             branches:
               ignore:
+                - master
                 - /docs-.*/
                 - /i18n-.*/
                 - /update-builder-.*/
           requires:
             - lint
       - static-analysis-and-no-known-cves:
+          filters:
+            branches:
+              ignore:
+                - master
+                - /docs-.*/
           requires:
             - lint
       - staging-test-with-rebase:
           filters:
             branches:
               ignore:
+                - master
                 - /docs-.*/
                 - /i18n-.*/
           requires:
             - lint
       - translation-tests:
+          filters:
+            branches:
+              ignore:
+                - master
+                - /docs-.*/
           requires:
             - lint
       - deb-tests:
@@ -369,5 +394,5 @@ workflows:
     jobs:
       - deb-tests
       - translation-tests
-      - docs-linkcheck
+      - docs-lint-linkcheck
       - fetch-tor-debs

.gitignore

@@ -72,7 +72,6 @@ lib/
 lib64/
 parts/
 sdist/
-var/
 wheels/
 *.egg-info/
 .installed.cfg

admin/securedrop_admin/__init__.py

@@ -24,6 +24,7 @@
 """
 
 import argparse
+import ipaddress
 import logging
 import os
 import io
@@ -33,6 +34,7 @@
 import json
 import base64
 import prompt_toolkit
+from prompt_toolkit.document import Document
 from prompt_toolkit.validation import Validator, ValidationError
 import yaml
 from pkg_resources import parse_version
@@ -48,6 +50,9 @@
 EXIT_SUBPROCESS_ERROR = 1
 EXIT_INTERRUPT = 2
 
+MAX_NAMESERVERS = 3
+LIST_SPLIT_RE = re.compile(r"\s*,\s*|\s+")
+
 
 class FingerprintException(Exception):
     pass
@@ -83,11 +88,34 @@ def validate(self, document):
 
     class ValidateIP(Validator):
         def validate(self, document):
-            if re.match(r'((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$',  # lgtm [py/regex/unmatchable-dollar] # noqa: E501
-                        document.text):
+            try:
+                ipaddress.ip_address(document.text)
                 return True
-            raise ValidationError(
-                message="An IP address must be something like 10.240.20.83")
+            except ValueError as e:
+                raise ValidationError(message=str(e))
+
+    class ValidateNameservers(Validator):
+        def validate(self, document):
+            candidates = LIST_SPLIT_RE.split(document.text)
+            if len(candidates) > MAX_NAMESERVERS:
+                raise ValidationError(message="Specify no more than three nameservers.")
+            try:
+                all(map(ipaddress.ip_address, candidates))
+            except ValueError:
+                raise ValidationError(
+                    message=(
+                        "DNS server(s) should be a space/comma-separated list "
+                        "of up to {} IP addresses"
+                    ).format(MAX_NAMESERVERS)
+                )
+            return True
+
+    @staticmethod
+    def split_list(text):
+        """
+        Splits a string containing a list of values separated by commas or whitespace.
+        """
+        return LIST_SPLIT_RE.split(text)
 
     class ValidatePath(Validator):
         def __init__(self, basedir):
@@ -273,10 +301,10 @@ def __init__(self, args):
              SiteConfig.ValidateNotEmpty(),
              None,
              lambda config: True],
-            ['dns_server', '8.8.8.8', str,
-             'DNS server specified during installation',
-             SiteConfig.ValidateNotEmpty(),
-             None,
+            ['dns_server', ['8.8.8.8', '8.8.4.4'], list,
+             'DNS server(s)',
+             SiteConfig.ValidateNameservers(),
+             SiteConfig.split_list,
              lambda config: True],
             ['securedrop_app_gpg_public_key', 'SecureDrop.asc', str,
              'Local filepath to public key for ' +
@@ -393,14 +421,15 @@ def __init__(self, args):
              lambda config: True],
         ]
 
-    def load_and_update_config(self):
+    def load_and_update_config(self, validate: bool = True, prompt: bool = True):
         if self.exists():
-            self.config = self.load()
+            self.config = self.load(validate)
 
-        return self.update_config()
+        return self.update_config(prompt)
 
-    def update_config(self):
-        self.config.update(self.user_prompt_config())
+    def update_config(self, prompt: bool = True):
+        if prompt:
+            self.config.update(self.user_prompt_config())
         self.save()
         self.validate_gpg_keys()
         self.validate_journalist_alert_email()
@@ -559,10 +588,56 @@ def save(self):
                            site_config_file,
                            default_flow_style=False)
 
-    def load(self):
+    def clean_config(self, config: dict) -> dict:
+        """
+        Cleans a loaded config without prompting.
+
+        For every variable defined in self.desc, validate its value in
+        the supplied configuration dictionary, run the value through
+        its defined transformer, and add the result to a clean version
+        of the configuration.
+
+        If no configuration variable triggers a ValidationError, the
+        clean configuration will be returned.
+        """
+        clean_config = {}
+        clean_config.update(config)
+        for desc in self.desc:
+            var, default, vartype, prompt, validator, transform, condition = desc
+            if var in clean_config:
+                value = clean_config[var]
+                if isinstance(value, list):
+                    text = " ".join(str(v) for v in value)
+                elif isinstance(value, bool):
+                    text = "yes" if value else "no"
+                else:
+                    text = str(value)
+
+                if validator is not None:
+                    try:
+                        validator.validate(Document(text))
+                    except ValidationError as e:
+                        sdlog.error(e)
+                        sdlog.error(
+                            'Error loading configuration. '
+                            'Please run "securedrop-admin sdconfig" again.'
+                        )
+                        raise
+                clean_config[var] = transform(text) if transform else text
+        return clean_config
+
+    def load(self, validate=True):
+        """
+        Loads the site configuration file.
+
+        If validate is True, then each configuration variable that has
+        an entry in self.desc is validated and transformed according
+        to current specifications.
+        """
         try:
             with io.open(self.args.site_config) as site_config_file:
-                return yaml.safe_load(site_config_file)
+                c = yaml.safe_load(site_config_file)
+                return self.clean_config(c) if validate else c
         except IOError:
             sdlog.error("Config file missing, re-run with sdconfig")
             raise
@@ -586,7 +661,7 @@ def setup_logger(verbose=False):
 
 def sdconfig(args):
     """Configure SD site settings"""
-    SiteConfig(args).load_and_update_config()
+    SiteConfig(args).load_and_update_config(validate=False)
     return 0
 
 
@@ -659,7 +734,7 @@ def find_or_generate_new_torv3_keys(args):
 
 def install_securedrop(args):
     """Install/Update SecureDrop"""
-    SiteConfig(args).load()
+    SiteConfig(args).load_and_update_config(prompt=False)
 
     sdlog.info("Now installing SecureDrop on remote servers.")
     sdlog.info("You will be prompted for the sudo password on the "

admin/tests/files/site-specific

@@ -1,6 +1,7 @@
 app_hostname: app
 app_ip: 10.20.2.2
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
 monitor_hostname: mon
 monitor_ip: 10.20.3.2
 ossec_alert_email: la@foo.com
@@ -13,7 +14,7 @@ securedrop_app_gpg_fingerprint: E99FFE83DF73E72FB6B264ED992D23B392F9E4F2
 securedrop_app_gpg_public_key: key.asc
 securedrop_app_https_on_source_interface: false
 securedrop_supported_locales:
-- en
+- en_US
 smtp_relay: smtp.gmail.com
 smtp_relay_port: 587
 ssh_users: sd

admin/tests/files/site-specific-missing-entries

@@ -1,6 +1,7 @@
 app_hostname: app
 app_ip: 10.20.2.2
-dns_server: 8.8.8.8
+dns_server:
+  - 8.8.8.8
 monitor_hostname: mon
 monitor_ip: 10.20.3.2
 ossec_alert_email: la@foo.com

admin/tests/test_integration.py

@@ -19,7 +19,9 @@
 OUTPUT1 = '''app_hostname: app
 app_ip: 10.20.2.2
 daily_reboot_time: 5
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
+- 8.8.4.4
 enable_ssh_over_tor: true
 journalist_alert_email: ''
 journalist_alert_gpg_public_key: ''
@@ -51,7 +53,9 @@
 WHEN_BOTH_TRUE = '''app_hostname: app
 app_ip: 10.20.2.2
 daily_reboot_time: 5
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
+- 8.8.4.4
 enable_ssh_over_tor: true
 journalist_alert_email: ''
 journalist_alert_gpg_public_key: ''
@@ -83,7 +87,9 @@
 WHEN_ONLY_V2 = '''app_hostname: app
 app_ip: 10.20.2.2
 daily_reboot_time: 5
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
+- 8.8.4.4
 enable_ssh_over_tor: true
 journalist_alert_email: ''
 journalist_alert_gpg_public_key: ''
@@ -115,7 +121,9 @@
 JOURNALIST_ALERT_OUTPUT = '''app_hostname: app
 app_ip: 10.20.2.2
 daily_reboot_time: 5
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
+- 8.8.4.4
 enable_ssh_over_tor: true
 journalist_alert_email: test@gmail.com
 journalist_alert_gpg_public_key: sd_admin_test.pub
@@ -147,7 +155,9 @@
 HTTPS_OUTPUT = '''app_hostname: app
 app_ip: 10.20.2.2
 daily_reboot_time: 5
-dns_server: 8.8.8.8
+dns_server:
+- 8.8.8.8
+- 8.8.4.4
 enable_ssh_over_tor: true
 journalist_alert_email: test@gmail.com
 journalist_alert_gpg_public_key: sd_admin_test.pub
@@ -229,8 +239,8 @@ def verify_hostname_mon_prompt(child):
 
 
 def verify_dns_prompt(child):
-    child.expect(rb'DNS server specified during installation\:', timeout=2)
-    assert ANSI_ESCAPE.sub('', child.buffer.decode("utf-8")).strip() == '8.8.8.8'  # noqa: E501
+    child.expect(rb'DNS server\(s\):', timeout=2)
+    assert ANSI_ESCAPE.sub('', child.buffer.decode("utf-8")).strip() == '8.8.8.8 8.8.4.4'  # noqa: E501
 
 
 def verify_app_gpg_key_prompt(child):
@@ -674,6 +684,7 @@ def test_sdconfig_enable_https_on_source_interface():
 
 @pytest.fixture
 def securedrop_git_repo(tmpdir):
+    cwd = os.getcwd()
     os.chdir(str(tmpdir))
     # Clone the SecureDrop repository into the temp directory.
     cmd = ['git', 'clone',
@@ -698,6 +709,8 @@ def securedrop_git_repo(tmpdir):
         # It means the coverage file may not exist, don't error
         pass
 
+    os.chdir(cwd)
+
 
 def set_reliable_keyserver(gpgdir):
     # If gpg.conf doesn't exist, create it and set a reliable default