freedomofpress · heartsucker · Sep 5, 2018 · Sep 9, 2018 · Sep 9, 2018 · Sep 19, 2018
@@ -26,112 +26,24 @@
     - gpg
     - securedrop_config
 
-  # The securedrop-app-code apt package contains only 'config.py.example',
-  # not 'config.py'. This was done so the securedrop-app-code
-  # package would not overwrite the config.py on upgrades.
-  # If 'config.py' exists, then the SecureDrop application
-  # has already been configured, and most of the following tasks will be skipped.
-- name: Check whether SecureDrop config.py file already exists.
-  stat:
-    path: "{{ securedrop_code }}/config.py"
-  register: config
-  # Read-only task, so don't report as changed.
-  changed_when: false
-  tags:
-    - securedrop_config
-
-- name: Copy starter config.py template if missing.
-  command: cp {{ securedrop_code }}/config.py.example {{ securedrop_code }}/config.py
-  when: not config.stat.exists
+- name: Migrate and populate the config files
+  command: >
+    /var/www/securedrop/populate_config.py \
+      --journalist-key-fpr {{ journalist_gpg_fpr }} \
+      --supported-locales '{{ securedrop_supported_locales|join(",") }}' \
+      --default-locale {{ securedrop_default_locale }}
   tags:
     - securedrop_config
 
-- name: Set ownership and permissions on config.py.
+- name: Set permissions on the config files
   file:
-    dest: "{{ securedrop_code }}/config.py"
-    owner: "{{ securedrop_user }}"
-    group: "{{ securedrop_user }}"
-    mode: "0600"
-  tags:
-    - permissions
-    - securedrop_config
-
-# Note: we can also use register with with_items to cut down on repetition
-# here. See
-# http://docs.ansible.com/playbooks_loops.html#using-register-with-a-loop
-
-- name: Generate 32-byte value for "source secret key".
-  shell: "head -c 32 /dev/urandom | base64"
-  register: source_secret_key
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Add 32-byte value for "source secret key" to config.py.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    regexp: "source_secret_key"
-    line: "    SECRET_KEY = '{{ source_secret_key.stdout}}'"
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Generate 32-byte value for "journalist secret key".
-  shell: "head -c 32 /dev/urandom | base64"
-  register: journalist_secret_key
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Add 32-byte value for "journalist secret key" to config.py.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    regexp: "journalist_secret_key"
-    line: "    SECRET_KEY = '{{ journalist_secret_key.stdout }}'"
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Generate 32-byte value for "scrypt id pepper".
-  shell: "head -c 32 /dev/urandom | base64"
-  register: scrypt_id_pepper
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Add 32-byte value for "scrypt id pepper" to config.py.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    regexp: "scrypt_id_pepper"
-    line: "SCRYPT_ID_PEPPER = '{{ scrypt_id_pepper.stdout }}'"
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Generate 32-byte value for "scrypt gpg pepper".
-  shell: "head -c 32 /dev/urandom | base64"
-  register: scrypt_gpg_pepper
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Add 32-byte value for "scrypt gpg pepper" to config.py.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    regexp: "scrypt_gpg_pepper"
-    line: "SCRYPT_GPG_PEPPER = '{{ scrypt_gpg_pepper.stdout }}'"
-  when: not config.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Declare Application GPG fingerprint in config.py.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    regexp: "^JOURNALIST_KEY = "
-    line: "JOURNALIST_KEY = '{{ securedrop_app_gpg_fingerprint }}'"
-  tags:
-    - gpg
-    - securedrop_config
+    path: "/etc/securedrop/{{ item }}"
+    owner: root
+    group: www-data
+    mode: "0750"
+  with_items:
+    - source-config.json
+    - journalist-config.json
 
 - name: Check whether sqlite database exists.
   stat:
@@ -151,21 +63,3 @@
   tags:
     - database
     - securedrop_config
-
-- name: Add DEFAULT_LOCALE to config.py if missing.
-  lineinfile:
-    dest: "{{ securedrop_code }}/config.py"
-    line: "DEFAULT_LOCALE = '{{ securedrop_default_locale }}'"
-    regexp: "^DEFAULT_LOCALE"
-  when: db.stat.exists
-  tags:
-    - securedrop_config
-
-- name: Update SUPPORTED_LOCALES in config.py
-  blockinfile:
-    content: "SUPPORTED_LOCALES = {{ securedrop_supported_locales + ['en_US'] }}"
-    path: "{{ securedrop_code }}/config.py"
-    marker: "# {mark} subset of the available locales that will be proposed to the user"
-  when: securedrop_supported_locales is defined
-  tags:
-    - securedrop_config
@@ -10,6 +10,16 @@
 import tarfile
 
 
+def try_add(backup, file_path):
+    try:
+        backup.add(file_path)
+    except OSError as e:
+        if e.errno == 2:  # file not found
+            print('File missing: {}'.format(file_path))
+        else:
+            raise e
+
+
 def main():
     backup_filename = 'sd-backup-{}.tar.gz'.format(
         datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
@@ -18,14 +28,23 @@ def main():
     sd_data = '/var/lib/securedrop'
 
     sd_code = '/var/www/securedrop'
-    sd_config = os.path.join(sd_code, "config.py")
+    sd_config_py = os.path.join(sd_code, "config.py")
+    sd_source_config = "/etc/securedrop/source-config.json"
+    sd_journalist_config = "/etc/securedrop/journalist-config.json"
     sd_custom_logo = os.path.join(sd_code, "static/i/logo.png")
 
     tor_hidden_services = "/var/lib/tor/services"
     torrc = "/etc/tor/torrc"
 
+    all_configs = [sd_config_py, sd_source_config, sd_journalist_config]
+
     with tarfile.open(backup_filename, 'w:gz') as backup:
-        backup.add(sd_config)
+        # Some of these configs may not be present because a migration may
+        # have failed, so we want to avoid failing to do a backup if any
+        # aren't present.
+        for config_file in all_configs:
+            try_add(backup, config_file)
+
         backup.add(sd_custom_logo)
         backup.add(sd_data)
         backup.add(tor_hidden_services)

@@ -67,6 +67,9 @@
   /etc/magic r,
   /etc/mime.types r,
   /etc/services r,
+  /etc/securedrop r,
+  /etc/securedrop/journalist-config.json r,
+  /etc/securedrop/source-config.json r,
   /etc/python2.7/sitecustomize.py r,
   /lib/x86_64-linux-gnu/libbz2.so.* mr,
   /lib/x86_64-linux-gnu/libc-*.so mr,
@@ -179,6 +182,8 @@
   /var/www/securedrop/journalist_templates/login.html r,
   /var/www/securedrop/journalist_templates/logo_upload_flashed.html r,
   /var/www/securedrop/journalist_templates/_source_row.html r,
+  /var/www/securedrop/populate_config.py r,
+  /var/www/securedrop/populate_config.pyc rw,
   /var/www/securedrop/models.py r,
   /var/www/securedrop/models.pyc rw,
   /var/www/securedrop/request_that_secures_file_uploads.py r,

@@ -6,10 +6,7 @@
     - pip
 
 - name: Compile PO to MO.
-  shell: >-
-    cp config.py.example config.py ;
-    trap 'rm config.py' EXIT ;
-    ./i18n_tool.py --verbose translate-messages --compile
+  shell: ./i18n_tool.py --verbose translate-messages --compile
   args:
     chdir: "{{ securedrop_code_filtered }}"
   environment:

@@ -47,6 +47,9 @@ def main():
     # Update the configs
     subprocess.check_call(['dpkg-reconfigure', 'securedrop-config'])
 
+    # Ensure that old backups are migrated to the new config format
+    subprocess.check_call(['/var/www/securedrop/populate_config.py'])
+
     # Reload Tor and the web server so they pick up the new configuration
     # If the process exits with a non-zero return code, raises an exception.
     subprocess.check_call(['service', 'apache2', 'restart'])

@@ -16,6 +16,18 @@
   async: 3600
   poll: 10
 
+# We do this here to ensure that if there was an error between config file
+# generation and permissions setting before the backup was run, we correct it.
+- name: Ensure correct permissions on the config files
+  file:
+    path: "/etc/securedrop/{{ item }}"
+    owner: root
+    group: www-data
+    mode: "0750"
+  with_items:
+    - source-config.json
+    - journalist-config.json
+
 - name: Delete the restore file to save disk space.
   file:
     path: /tmp/{{ restore_file }}

@@ -124,13 +124,20 @@ case "$1" in
 
     chown -R www-data:www-data /var/lib/securedrop /var/www/securedrop
 
+    mkdir -p /etc/securedrop
+    chmod 0750 /etc/securedrop
+
     pip install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
       -r /var/www/securedrop/requirements/securedrop-app-code-requirements.txt
 
     chown -R www-data:www-data /var/www/securedrop
     chown www-data:www-data /var/www/journalist.wsgi
     chown www-data:www-data /var/www/source.wsgi
 
+    # Migrate the python-based config to JSON
+    /var/www/securedrop/populate_config.py
+    chown -R root:www-data /etc/securedrop
+
     # Apache's default sites are not allowed by the securedrop apparmor profile
     # disable the site before putting the apache apparmor profile in enforce
     # mode.

@@ -156,15 +156,17 @@ def test_deb_package_control_fields_homepage(host, deb):
 def test_deb_package_contains_no_config_file(host, deb):
     """
     Ensures the `securedrop-app-code` package does not ship a `config.py`
-    file. Doing so would clobber the site-specific changes made via Ansible.
+    file or `{source,journalist}-config.json`. Doing so would clobber the
+    site-specific changes made via Ansible.
 
     Somewhat lazily checking all deb packages, rather than just the app-code
     package, but it accomplishes the same in a DRY manner.
     """
     deb_package = host.file(deb.format(
         securedrop_test_vars.securedrop_version))
     c = host.run("dpkg-deb --contents {}".format(deb_package.path))
-    assert not re.search("^.*/config\.py$", c.stdout, re.M)
+    assert not re.search("^.*/(source-|journalist-|)config\.(py|json)$",
+                         c.stdout, re.M)
 
 
 @pytest.mark.parametrize("deb", deb_packages)

@@ -1,3 +1,4 @@
+import json
 import pytest
 
 
@@ -34,18 +35,6 @@ def test_securedrop_application_apt_dependencies(host, package):
     assert host.package(package).is_installed
 
 
-def test_securedrop_application_test_locale(host):
-    """
-    Ensure SecureDrop DEFAULT_LOCALE is present.
-    """
-    securedrop_config = host.file("{}/config.py".format(
-        securedrop_test_vars.securedrop_code))
-    with host.sudo():
-        assert securedrop_config.is_file
-        assert securedrop_config.contains("^DEFAULT_LOCALE")
-        assert securedrop_config.content.count("DEFAULT_LOCALE") == 1
-
-
 def test_securedrop_application_test_journalist_key(host):
     """
     Ensure the SecureDrop Application GPG public key file is present.
@@ -64,17 +53,20 @@ def test_securedrop_application_test_journalist_key(host):
 
     # Let's make sure the corresponding fingerprint is specified
     # in the SecureDrop app configuration.
-    securedrop_config = host.file("{}/config.py".format(
-        securedrop_test_vars.securedrop_code))
-    with host.sudo():
-        assert securedrop_config.is_file
-        assert securedrop_config.user == \
-            securedrop_test_vars.securedrop_user
-        assert securedrop_config.group == \
-            securedrop_test_vars.securedrop_user
-        assert oct(securedrop_config.mode) == "0600"
-        assert securedrop_config.contains(
-            "^JOURNALIST_KEY = '65A1B5FF195B56353CC63DFFCC40EF1228271441'$")
+    for config_name in ['source', 'journalist']:
+        config_file = File("/etc/securedrop/{}-config.json"
+                           .format(config_name))
+        with host.sudo():
+            assert config_file.is_file
+            assert config_file.user == 'root'
+            assert config_file.group == securedrop_test_vars.securedrop_user
+            assert oct(config_file.mode) == "0640"
+
+            with open(config_file) as f:
+                config_json = json.loads(f.read())
+
+        assert config_json['journalist_key'] == \
+            '65A1B5FF195B56353CC63DFFCC40EF1228271441'
 
 
 def test_securedrop_application_sqlite_db(host):

@@ -32,22 +32,6 @@ translation-test: ## Run all pages-layout tests in all supported languages
 dev: ## Run the dev server
 	 DOCKER_RUN_ARGUMENTS='-p127.0.0.1:8080:8080 -p127.0.0.1:8081:8081 -p127.0.0.1:5901:5901' ./bin/dev-shell ./bin/run
 
-config.py: test-config
-
-.PHONY: test-config
-test-config: ## Generate the test config
-	@source_secret_key=$(shell head -c 32 /dev/urandom | base64) \
-	journalist_secret_key=$(shell head -c 32 /dev/urandom | base64) \
-	scrypt_id_pepper=$(shell head -c 32 /dev/urandom | base64) \
-	scrypt_gpg_pepper=$(shell head -c 32 /dev/urandom | base64) \
-	python -c 'import os; from jinja2 import Environment, FileSystemLoader; \
-         env = Environment(loader=FileSystemLoader(".")); \
-         ctx = {"securedrop_app_gpg_fingerprint": "65A1B5FF195B56353CC63DFFCC40EF1228271441"}; \
-         ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.iteritems())); \
-         ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
-	@echo >> config.py
-	@echo "SUPPORTED_LOCALES = ['ar', 'de_DE', 'es_ES', 'en_US', 'el', 'fr_FR', 'it_IT', 'nb_NO', 'nl', 'pt_BR', 'tr', 'zh_Hant']"  >> config.py
-
 .PHONY: update-user-guides
 update-user-guides: ## Runs the page layout tests to regenerate screenshots
 	./bin/dev-shell ./bin/update-user-guides

@@ -20,7 +20,8 @@
     # These imports are only needed for offline generation of automigrations.
     # Importing them in a prod-like environment breaks things.
     from journalist_app import create_app  # noqa
-    from sdconfig import config as sdconfig  # noqa
+    from sdconfig import JournalistInterfaceConfig  # noqa
+    sdconfig = JournalistInterfaceConfig()
 
     # App context is needed for autogenerated migrations
     create_app(sdconfig).app_context().push()