Merge pull request #3619 from freedomofpress/journalist-api-0.9.0

Add journalist interface API

docs/index.rst

@@ -104,6 +104,7 @@ anonymous sources.
    development/making_pr
    development/admin_development
    development/updategui_development
+   development/journalist_api
    development/virtual_environments
    development/virtualizing_tails
    development/contributor_guidelines

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2

@@ -141,6 +141,8 @@
   /var/www/securedrop/journalist_app/account.pyc rw,
   /var/www/securedrop/journalist_app/admin.py r,
   /var/www/securedrop/journalist_app/admin.pyc rw,
+  /var/www/securedrop/journalist_app/api.py r,
+  /var/www/securedrop/journalist_app/api.pyc rw,
   /var/www/securedrop/journalist_app/col.py r,
   /var/www/securedrop/journalist_app/col.pyc rw,
   /var/www/securedrop/journalist_app/decorators.py r,

securedrop/alembic/env.py

@@ -68,7 +68,8 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

securedrop/alembic/versions/3d91d6948753_create_source_uuid_column.py

@@ -0,0 +1,69 @@
+"""Create source UUID column
+
+Revision ID: 3d91d6948753
+Revises: faac8092c123
+Create Date: 2018-07-09 22:39:05.088008
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import quoted_name
+import subprocess
+import uuid
+
+# revision identifiers, used by Alembic.
+revision = '3d91d6948753'
+down_revision = 'faac8092c123'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Schema migration
+    op.rename_table('sources', 'sources_tmp')
+
+    # Add UUID column.
+    op.add_column('sources_tmp', sa.Column('uuid', sa.String(length=36)))
+
+    # Add UUIDs to sources_tmp table.
+    conn = op.get_bind()
+    sources = conn.execute(sa.text("SELECT * FROM sources_tmp")).fetchall()
+
+    for source in sources:
+        id = source.id
+        source_uuid = str(uuid.uuid4())
+        conn.execute(
+            sa.text("UPDATE sources_tmp SET uuid=('{}') WHERE id={}".format(
+                source_uuid, id)))
+
+    # Now create new table with unique constraint applied.
+    op.create_table(quoted_name('sources', quote=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('filesystem_id', sa.String(length=96), nullable=True),
+        sa.Column('journalist_designation', sa.String(length=255),
+                  nullable=False),
+        sa.Column('flagged', sa.Boolean(), nullable=True),
+        sa.Column('last_updated', sa.DateTime(), nullable=True),
+        sa.Column('pending', sa.Boolean(), nullable=True),
+        sa.Column('interaction_count', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('uuid'),
+        sa.UniqueConstraint('filesystem_id')
+    )
+
+    # Data Migration: move all sources into the new table.
+    conn.execute('''
+        INSERT INTO sources
+        SELECT id, uuid, filesystem_id, journalist_designation, flagged,
+               last_updated, pending, interaction_count
+        FROM sources_tmp
+    ''')
+
+    # Now delete the old table.
+    op.drop_table('sources_tmp')
+
+
+def downgrade():
+    with op.batch_alter_table('sources', schema=None) as batch_op:
+        batch_op.drop_column('uuid')

securedrop/alembic/versions/fccf57ceef02_create_submission_uuid_column.py

@@ -0,0 +1,66 @@
+"""create submission uuid column
+
+Revision ID: fccf57ceef02
+Revises: 3d91d6948753
+Create Date: 2018-07-12 00:06:20.891213
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+import uuid
+
+# revision identifiers, used by Alembic.
+revision = 'fccf57ceef02'
+down_revision = '3d91d6948753'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Schema migration
+    op.rename_table('submissions', 'submissions_tmp')
+
+    # Add UUID column.
+    op.add_column('submissions_tmp', sa.Column('uuid', sa.String(length=36)))
+
+    # Add UUIDs to submissions_tmp table.
+    conn = op.get_bind()
+    submissions = conn.execute(
+        sa.text("SELECT * FROM submissions_tmp")).fetchall()
+
+    for submission in submissions:
+        id = submission.id
+        submission_uuid = str(uuid.uuid4())
+        conn.execute(
+            sa.text("""UPDATE submissions_tmp
+                       SET uuid=('{}')
+                       WHERE id={}""".format(submission_uuid, id)))
+
+    # Now create new table with unique constraint applied.
+    op.create_table('submissions',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('source_id', sa.Integer(), nullable=True),
+        sa.Column('filename', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer(), nullable=False),
+        sa.Column('downloaded', sa.Boolean(), nullable=True),
+        sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('uuid')
+    )
+
+    # Data Migration: move all submissions into the new table.
+    conn.execute('''
+        INSERT INTO submissions
+        SELECT id, uuid, source_id, filename, size, downloaded
+        FROM submissions_tmp
+    ''')
+
+    # Now delete the old table.
+    op.drop_table('submissions_tmp')
+
+
+def downgrade():
+    with op.batch_alter_table('submissions', schema=None) as batch_op:
+        batch_op.drop_column('uuid')

securedrop/crypto_util.py

@@ -191,6 +191,10 @@ def getkey(self, name):
                     return key['fingerprint']
         return None
 
+    def export_pubkey(self, name):
+        fingerprint = self.getkey(name)
+        return self.gpg.export_keys(fingerprint)
+
     def encrypt(self, plaintext, fingerprints, output=None):
         # Verify the output path
         if output:

securedrop/journalist_app/__init__.py

@@ -1,19 +1,21 @@
 # -*- coding: utf-8 -*-
 
 from datetime import datetime, timedelta
-from flask import Flask, session, redirect, url_for, flash, g, request
+from flask import (Flask, session, redirect, url_for, flash, g, request,
+                   render_template)
 from flask_assets import Environment
 from flask_babel import gettext
 from flask_wtf.csrf import CSRFProtect, CSRFError
 from os import path
+from werkzeug.exceptions import default_exceptions  # type: ignore
 
 import i18n
 import template_filters
 import version
 
 from crypto_util import CryptoUtil
 from db import db
-from journalist_app import account, admin, main, col
+from journalist_app import account, admin, api, main, col
 from journalist_app.utils import get_source, logged_in
 from models import Journalist
 from store import Storage
@@ -39,7 +41,7 @@ def create_app(config):
     app.config.from_object(config.JournalistInterfaceFlaskConfig)
     app.sdconfig = config
 
-    CSRFProtect(app)
+    csrf = CSRFProtect(app)
     Environment(app)
 
     if config.DATABASE_ENGINE == "sqlite":
@@ -80,6 +82,18 @@ def handle_csrf_error(e):
         flash(msg, 'error')
         return redirect(url_for('main.login'))
 
+    def _handle_http_exception(error):
+        # Workaround for no blueprint-level 404/5 error handlers, see:
+        # https://github.com/pallets/flask/issues/503#issuecomment-71383286
+        handler = app.error_handler_spec['api'][error.code].values()[0]
+        if request.path.startswith('/api/') and handler:
+            return handler(error)
+
+        return render_template('error.html', error=error), error.code
+
+    for code in default_exceptions:
+        app.errorhandler(code)(_handle_http_exception)
+
     i18n.setup_app(config, app)
 
     app.jinja_env.trim_blocks = True
@@ -97,17 +111,6 @@ def handle_csrf_error(e):
         template_filters.rel_datetime_format
     app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat
 
-    @app.template_filter('autoversion')
-    def autoversion_filter(filename):
-        """Use this template filter for cache busting"""
-        absolute_filename = path.join(config.SECUREDROP_ROOT, filename[1:])
-        if path.exists(absolute_filename):
-            timestamp = str(path.getmtime(absolute_filename))
-        else:
-            return filename
-        versioned_filename = "{0}?v={1}".format(filename, timestamp)
-        return versioned_filename
-
     @app.before_request
     def setup_g():
         """Store commonly used values in Flask's special g object"""
@@ -130,8 +133,11 @@ def setup_g():
         g.html_lang = i18n.locale_to_rfc_5646(g.locale)
         g.locales = i18n.get_locale2name()
 
-        if request.endpoint not in _insecure_views and not logged_in():
-            return redirect(url_for('main.login'))
+        if request.path.split('/')[1] == 'api':
+            pass  # We use the @token_required decorator for the API endpoints
+        else:  # We are not using the API
+            if request.endpoint not in _insecure_views and not logged_in():
+                return redirect(url_for('main.login'))
 
         if request.method == 'POST':
             filesystem_id = request.form.get('filesystem_id')
@@ -144,5 +150,8 @@ def setup_g():
                            url_prefix='/account')
     app.register_blueprint(admin.make_blueprint(config), url_prefix='/admin')
     app.register_blueprint(col.make_blueprint(config), url_prefix='/col')
+    api_blueprint = api.make_blueprint(config)
+    app.register_blueprint(api_blueprint, url_prefix='/api/v1')
+    csrf.exempt(api_blueprint)
 
     return app