Templating, Formio Attachments, Logging

.circleci/config.yml

@@ -9,7 +9,12 @@ jobs:
       # specify the version you desire here
       # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
       - image: circleci/python:3.7.2
-
+        environment:
+          DATABASE_URL: postgresql://circleci@127.0.0.1:5432/circle_test
+      - image: circleci/postgres:11.2
+        environment:
+          POSTGRES_USER: circleci
+          POSTGRES_DB: circle_test
       # Specify service dependencies here if necessary
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
@@ -31,7 +36,7 @@ jobs:
           name: install dependencies
           command: |
             sudo pip install pipenv
-            pipenv install
+            pipenv install --dev
 
       - save_cache:
           paths:
@@ -43,15 +48,18 @@ jobs:
           name: run pre-commit
           command: |
             pipenv run pre-commit run --all-files
-
+
+      - run:
+          name: run db migrations
+          command: |
+            pipenv run alembic upgrade head
+
       - run:
           name: start gunicorn
           command: |
             pipenv run gunicorn 'service.microservice:start_service()'
           background: true
 
-
-
       # run tests!
       # this example uses Django's built-in test-runner
       # other common Python testing frameworks include pytest and nose

.env.example

@@ -0,0 +1,5 @@
+export ACCESS_KEY=123456
+export SENTRY_DSN=
+
+export DATABASE_URL=postgresql://localhost/email_microservice
+export SENDGRID_API_KEY=abc123

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+repos:
 -   repo: local
     hooks:
     - id: pylint

Pipfile

@@ -18,6 +18,11 @@ pre-commit = "*"
 pylint = "*"
 sendgrid = "*"
 urllib3 = ">=1.26.5"
+jinja2 = "*"
+sqlalchemy = "*"
+psycopg2 = "*"
+alembic = "*"
+bs4 = "*"
 
 [requires]
 python_version = "3.7"

Procfile

@@ -1 +1,2 @@
-web: gunicorn 'service.microservice:start_service()' --log-file -
+web: bin/qgtunnel pipenv run gunicorn 'service.microservice:start_service()'
+release: bin/qgtunnel pipenv run alembic upgrade head

README.md

@@ -62,44 +62,27 @@ For detail of the fields in the json data, please see below.
   |enable | Indicates if this setting is enabled.| optional |
   |post_to_url | An Inbound Parse URL that you would like a copy of your email along with the spam report to be sent to.| optional |
   | threshold | The threshold used to determine if your content qualifies as spam on a scale from 1 to 10, with 10 being most strict, or most likely to be considered as spam.| optional |
-|personalizations | An array of messages and their metadata. Each object within personalizations can be thought of as an envelope - it defines who should receive an individual message and how that message should be handled.| ||required |
- |to| An array of recipients. Each object within this array may contain the name, but must always contain the email, of a recipient.|| required|
-  |email | Email | required |
-  |name | The name of the person or company that is sending the email.| optional|
- |bcc | An array of recipients who will receive a blind carbon copy of your email. Each object within this array may contain the name, but must always contain the email, of a recipient.| |optional |
-  |email | Email | required |
-  |name | The name of the person or company that is sending the email. | optional|
- |cc | An array of recipients who will receive a copy of your email. Each object within this array may contain the name, but must always contain the email, of a recipient.|| optional |
-  |email | Email | required |
-  |name | The name of the person or company that is sending the email. | optional|
- |custom_args| Values that are specific to this personalization that will be carried along with the email and its activity data. Substitutions will not be made on custom arguments, so any string that is entered into this parameter will be assumed to be the custom argument that you would like to be used. May not exceed 10,000 bytes.| | optional |
- |headers| A collection of JSON key/value pairs allowing you to specify specific handling instructions for your email. You may not overwrite the following headers: x-sg-id, x-sg-eid, received, dkim-signature, Content-Type, Content-Transfer-Encoding, To, From, Subject, Reply-To, CC, BCC || optional|
- |subject |The subject of your email. Char length requirements, according to the RFC - http://stackoverflow.com/questions/1592291/what-is-the-email-subject-length-limit#answer-1592310 | | required |
 |reply_to | | required |
  |email | Email | | required |
  |name | The name of the person or company that is sending the email. || optional|
 |sections| An object of key/value pairs that define block sections of code to be used as substitutions. The key/value pairs must be strings. ||| optional|
-|send_at | A unix timestamp allowing you to specify when you want your email to be delivered. This may be overridden by the personalizations[x].send_at parameter. You can't schedule more than 72 hours in advance. If you have the flexibility, it's better to schedule mail for off-peak times. Most emails are scheduled and sent at the top of the hour or half hour. Scheduling email to avoid those times (for example, scheduling at 10:53) can result in lower deferral rates because it won't be going through our servers at the same times as everyone else's mail.||| optional |
 |subject| The subject of your email. Char length requirements, according to the RFC - http://stackoverflow.com/questions/1592291/what-is-the-email-subject-length-limit#answer-1592310| ||required|
-|template_id | The id of a template that you would like to use. If you use a template that contains a subject and content (either text or html), you do not need to specify those at the personalizations nor message level.| ||optional |
-|dynamic_template_data| If `template_id` is specified, the key/value pair will be mapped in the template. See [Dynamic Template Data](https://sendgrid.com/docs/ui/sending-email/how-to-send-an-email-with-dynamic-transactional-templates/) for more details.| || optional |
-|tracking_settings | Settings to determine how you would like to track the metrics of how your recipients interact with your email.| || optional |
- |click_tracking| Allows you to track whether a recipient clicked a link in your email.|| optional|
-  |enable| Indicates if this setting is enabled.| optional|
-  |enable_text|Indicates if this setting should be included in the text/plain portion of your email. | optional|
- |ganalytics |Allows you to enable tracking provided by Google Analytics. | | optional|
-  |enable | Indicates if this setting is enabled.| optional|
-  |utm_source Name of the referrer source. (e.g. Google SomeDomain.com or Marketing Email) | optional|
-  |utm_medium NAME OF YOUR MARKETING MEDIUM e.g. email| optional|
-  |utm_term IDENTIFY PAID KEYWORDS HERE| optional|
-  |utm_content USE THIS SPACE TO DIFFERENTIATE YOUR EMAIL FROM ADS| optional|
-  |utm_campaign The name of the campaign| optional|
- |open_tracking |Allows you to track whether the email was opened or not, but including a single pixel image in the body of the content. When the pixel is loaded, we can log that the email was opened. | | optional |
-  |enable | Indicates if this setting is enabled.| optional|
-  |substitution_tag| Allows you to specify a substitution tag that you can insert in the body of your email at a location that you desire. This tag will be replaced by the open tracking pixel. | optional|
- |subscription_tracking |Allows you to insert a subscription management link at the bottom of the text and html bodies of your email. If you would like to specify the location of the link within your email, you may use the substitution_tag. | | optional|
-  |enable | Indicates if this setting is enabled.| optional|
-&nbsp;&nbsp;|html | HTML to be appended to the email, with the subscription tracking link. You may control where the link is by using the tag <% %>| optional| If you would like to unsubscribe and stop receiving these emails <% clickhere %>.
-&nbsp;&nbsp;|substitution_tag| A tag that will be replaced with the unsubscribe URL. for example: [unsubscribe_url]. If this parameter is used, it will override both the text and html parameters. The URL of the link will be placed at the substitution tag’s location, with no additional formatting. | optional|
-&nbsp;&nbsp;|text | Text to be appended to the email, with the subscription tracking link. You may control where the link is by using the tag <% %>| optional|
 
+## Testing
+Code coverage command with missing statement line numbers
+```
+pipenv run python -m pytest -s --cov=service --cov=tasks tests/ --cov-report term-missing
+```
+
+## Revising the database
+Create a migration
+```
+pipenv run alembic revision -m "Add a column"
+```
+Edit the created revision file to add the steps to implement and rollback
+the changes you want to make.
+
+Run DB migrations
+```
+pipenv run alembic upgrade head
+```

alembic.ini

@@ -0,0 +1,100 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator"
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. Valid values are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # default: use os.pathsep
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app.json

@@ -0,0 +1,27 @@
+{
+    "addons": [
+      {
+        "plan": "papertrail:choklad"
+      },
+      {
+        "plan": "quotaguardstatic:starter"
+      }
+    ],
+    "buildpacks": [
+      {
+        "url": "heroku/python"
+      },
+      {
+        "url": "https://github.com/SFDigitalServices/heroku-configvar-files-buildpack"
+      }
+    ],
+    "env": {
+    },
+    "formation": {
+    },
+    "name": "email-microservice",
+    "scripts": {
+      "postdeploy": "pipenv run alembic upgrade head"
+    },
+    "stack": "heroku-20"
+  }