Merge pull request #12 from alphagov/pmi-753-env-var-for-google-sheet

PMI-753: Read Google auth credentials from environment variables

.env.example

@@ -1,3 +1,8 @@
 AWS_PROFILE=verify-audit-billing-dev
 RP_REPORT_OUTPUT_BUCKET=govukverify-hub-integration-billing-reports
 PIWIK_API_TOKEN=<your-Piwik-API-token>
+GOOGLE_AUTH_PRIVATE_KEY_ID=<your-Google-auth-private-key-id>
+# GOOGLE_AUTH_PRIVATE_KEY=(cannot be set in `.env.`, must be defined in shell)
+GOOGLE_AUTH_CLIENT_EMAIL=<your-Google-auth-client-email>
+GOOGLE_AUTH_CLIENT_ID=<your-Google-auth-client-ID>
+GOOGLE_AUTH_CLIENT_CERT_URL=<your-Google-auth-client-cert-URL>

performance/config.py

@@ -22,6 +22,8 @@ class Config:
     PIWIK_LIMIT = '-1'
     PIWIK_BASE_URL = 'https://analytics-hub-prod-a-dmz.ida.digital.cabinet-office.gov.uk/index.php'
     DEFAULT_OUTPUT_PATH = os.path.join(BASE_DIR, 'output')
+    # This is only used if Google auth credentials aren't already present in environment variables See
+    # `performance.gsheets.get_pygsheets_client` for implementation details.
     GSHEETS_CREDENTIALS_FILE = os.path.join(
         VERIFY_DATA_PIPELINE_CONFIG_PATH, 'credentials', 'google_sheets_credentials.json')
 

performance/gsheets.py

@@ -0,0 +1,35 @@
+import json
+import os
+import pygsheets
+import tempfile
+
+from performance.env import check_get_env
+from performance import prod_config as config
+
+
+def get_pygsheets_client():
+    google_auth_private_key_id = os.getenv('GOOGLE_AUTH_PRIVATE_KEY_ID')
+    if not google_auth_private_key_id:
+        return pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE)
+
+    creds = {
+        "type": "service_account",
+        "project_id": "verify-performance",
+        "private_key_id": google_auth_private_key_id,
+        # Private key cannot be set in `.env` at the moment and must therefore be defined in the
+        # shell when running locally.
+        "private_key": check_get_env('GOOGLE_AUTH_PRIVATE_KEY'),
+        "client_email": check_get_env('GOOGLE_AUTH_CLIENT_EMAIL'),
+        "client_id": check_get_env('GOOGLE_AUTH_CLIENT_ID'),
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+        "client_x509_cert_url": check_get_env('GOOGLE_AUTH_CLIENT_CERT_URL')
+    }
+    temp = tempfile.NamedTemporaryFile(delete=False, mode="w+t")
+    temp.write(json.dumps(creds))
+    temp.close()
+    try:
+        return pygsheets.authorize(service_file=temp.name)
+    finally:
+        os.unlink(temp.name)

performance/reports/rp.py

@@ -1,16 +1,14 @@
 import os
 
 import pandas
-import pygsheets
 
+import performance
 import performance.piwik as piwik
 import performance.billing as billing
-
 from performance import prod_config as config
-import performance
+from performance.gsheets import get_pygsheets_client
 from performance.reports.tests import conftest
 
-
 RP_REPORT_COLUMNS = [
     'rp',
     'all_referrals_with_intent',
@@ -74,7 +72,7 @@ def test_upload(gsheets_key, date_start):
     # because if you want to test the upload process, you don't want to have to wait around
     # for Piwik - any data will do.
     config = performance.config.TestConfig()
-    pygsheets_client = pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE)
+    pygsheets_client = get_pygsheets_client()
     for rp_info in config.rp_information.values():
         rp_info['sheet_key'] = gsheets_key
 
@@ -119,7 +117,7 @@ def export_metrics_to_csv(df_export, report_output_path, date_start):
 
 
 def export_metrics_to_google_sheets(df_export, date_start):
-    pygsheets_client = pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE)
+    pygsheets_client = get_pygsheets_client()
     exporter = GoogleSheetsRelyingPartyReportExporter(config, pygsheets_client)
     exporter.export(df_export, column_heading=date_start)