Making staging/production environments work

README.md

@@ -47,8 +47,8 @@ This project is aimed at UNIX runtimes, if you are on Windows, consider using [W
 **Runtime Bits**
 - Install the [Google Cloud SDK](https://cloud.google.com/sdk)
 - Setup your Google Account, following the [Serverless Google Cloud Functions Guide](https://www.serverless.com/framework/docs/providers/google/guide/credentials/)
-- Place your Google Account service key in the ROOT DIRECTORY OF YOUR TERMINAL RUNTIME `cd ~` named `gcloud-service-key.json` to match `serverless.yml`
-- Run `./local.sh api` to run the api function locally, and `./local.sh scrapers` to run the scraper function locally with hot-reload
+- Place your Google Account service key in the ROOT DIRECTORY OF YOUR TERMINAL RUNTIME `cd ~` named `./.gcloud/japan-grid-carbon-service-key-<environment>.json` to match `serverless.yml`
+- Run `./local.sh api staging` to run the api function locally, and `./local.sh scrapers staging` to run the scraper function locally with hot-reload in staging
 - Use cURL, Postman etc. and ping `http://localhost:8080/<etc>` to initiate the function
 
 ### Pumped Storage Problem
@@ -82,15 +82,16 @@ source .venv/bin/activate
 # Read the Recent Logs
 gcloud functions logs read
 
-# Deploy Endpoint
-./deployendpoint.sh daily_carbon_intensity
+# Deploy Manually
+#   First argument, environment
+#   Second argument, project ID in google cloud
+./deploy.sh staging japan-grid-carbon-api
 
 # Run Locally
-./local.sh api
-./local.sh scrapers
-
-# Deploy All
-./deployAll.sh
+#   First argument, api name
+#   Second argument, environment
+./local.sh api scraping
+./local.sh scrapers production
 
 # Run Tests
 pytest -vv

cloud_functions/api/test_main.py

@@ -1,6 +1,9 @@
 import pytest
 import json
 import gc
+import os
+os.environ["STAGE"] = "staging"
+
 from .main import (daily_carbon_intensity,
                    daily_carbon_intensity_with_breakdown,
                    daily_carbon_intensity_prediction,

cloud_functions/api/utilities/UtilityAPI.py

@@ -2,11 +2,14 @@
 import requests
 import json
 from google.cloud import bigquery
+import os
+stage = os.environ['STAGE']
 
 
 class UtilityAPI:
     def __init__(self, utility):
         self.utility = utility
+        self.bqStageName = "" if stage == "production" else "-staging"
 
     # Likely to be Overwritten
     def _get_intensity_query_string(self):
@@ -25,8 +28,9 @@ def _get_intensity_query_string(self):
             (if(kWh_interconnectors > 0,kWh_interconnectors, 0) * {intensity_interconnectors}) 
             ) / kWh_total
             ) as carbon_intensity
-        FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+        FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],
@@ -126,19 +130,20 @@ def _extract_prediction_from_big_query_by_weekday_month_and_year(self, year):
         SELECT
         predicted_carbon_intensity, year, dayofweek, month, hour
         FROM
-        ML.PREDICT(MODEL `japan-grid-carbon-api.{utility}.year_month_dayofweek_model`,
+        ML.PREDICT(MODEL `japan-grid-carbon-api{bqStageName}.{utility}.year_month_dayofweek_model`,
             (
             SELECT
                 {year} AS year,
                 EXTRACT(DAYOFWEEK FROM datetime) AS dayofweek,
                 EXTRACT(MONTH FROM datetime) AS month,
                 EXTRACT(HOUR FROM datetime) AS hour,
-            FROM japan-grid-carbon-api.{utility}.historical_data_by_generation_type
+            FROM japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type
             GROUP BY month, dayofweek, hour
             )
         )
         order by month, dayofweek, hour asc
         """.format(
+                bqStageName=self.bqStageName,
                 utility=self.utility,
                 year=year
         )
@@ -268,11 +273,14 @@ def create_linear_regression_model(self):
         client = bigquery.Client()
 
         query = """
-        CREATE OR REPLACE MODEL `japan-grid-carbon-api.{utility}.year_month_dayofweek_model`
+        CREATE OR REPLACE MODEL `japan-grid-carbon-api{bqStageName}.{utility}.year_month_dayofweek_model`
         OPTIONS(
         model_type='LINEAR_REG',
         input_label_cols=['carbon_intensity']
-        ) AS""".format(utility=self.utility) + """
+        ) AS""".format(
+            bqStageName=self.bqStageName,
+            utility=self.utility
+        ) + """
         SELECT
         EXTRACT(MONTH FROM datetime) AS month,
         EXTRACT(YEAR FROM datetime) AS year,

cloud_functions/api/utilities/cepco/CepcoAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/chuden/ChudenAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/hepco/HepcoAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/kepco/KepcoAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/kyuden/KyudenAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/okiden/OkidenAPI.py

@@ -23,9 +23,10 @@ def _get_intensity_query_string(self):
         FROM (
             SELECT *,
             (MWh_fossil + MWh_hydro + MWh_biomass + MWh_solar_output + MWh_wind_output) as MWh_total_generation
-            FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+            FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_fossil=ci["kWh_fossil"],
             intensity_hydro=ci["kWh_hydro"],

cloud_functions/api/utilities/rikuden/RikudenAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/tepco/TepcoAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(daMWh_interconnectors > 0,daMWh_interconnectors, 0) as daMWh_interconnector_contribution,
                 if(daMWh_pumped_storage > 0,daMWh_pumped_storage, 0) as daMWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/test_UtilityAPI.py

@@ -70,7 +70,7 @@ def test_gbq_query_string():
             (if(kWh_interconnectors > 0,kWh_interconnectors, 0) * 500) 
             ) / kWh_total
             ) as carbon_intensity
-        FROM `japan-grid-carbon-api.tepco.historical_data_by_generation_type`
+        FROM `japan-grid-carbon-api-staging.tepco.historical_data_by_generation_type`
         """
 
     assert expected == api._get_intensity_query_string()

cloud_functions/api/utilities/tohokuden/TohokudenAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(MWh_interconnectors > 0,MWh_interconnectors, 0) as MWh_interconnector_contribution,
                 if(MWh_pumped_storage > 0,MWh_pumped_storage, 0) as MWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/api/utilities/yonden/YondenAPI.py

@@ -29,10 +29,11 @@ def _get_intensity_query_string(self):
                 SELECT *,
                 if(daMWh_interconnectors > 0,daMWh_interconnectors, 0) as daMWh_interconnector_contribution,
                 if(daMWh_pumped_storage > 0,daMWh_pumped_storage, 0) as daMWh_pumped_storage_contribution,
-                FROM `japan-grid-carbon-api.{utility}.historical_data_by_generation_type`
+                FROM `japan-grid-carbon-api{bqStageName}.{utility}.historical_data_by_generation_type`
             )
         )
         """.format(
+            bqStageName=self.bqStageName,
             utility=self.utility,
             intensity_nuclear=ci["kWh_nuclear"],
             intensity_fossil=ci["kWh_fossil"],

cloud_functions/requirements.txt

@@ -53,6 +53,7 @@ requests-oauthlib==1.3.0
 rsa==4.6
 six==1.15.0
 toml==0.10.1
+tqdm==4.50.2
 urllib3==1.25.9
 watchdog==0.10.3
 Werkzeug==1.0.1

cloud_functions/scrapers/area_data/AreaDataScraper.py

@@ -1,9 +1,13 @@
 import json
+import os
+from datetime import datetime
 from google.cloud import storage
 from google.cloud import bigquery
 from google.api_core import retry
 from pydoc import locate
 
+stage = os.environ['STAGE']
+
 from scrapers.area_data.utilities.tepco.TepcoAreaScraper import TepcoAreaScraper
 from scrapers.area_data.utilities.kepco.KepcoAreaScraper import KepcoAreaScraper
 from scrapers.area_data.utilities.tohokuden.TohokudenAreaScraper import TohokudenAreaScraper
@@ -57,9 +61,12 @@ def scrape(self):
 
     def _upload_blob_to_storage(self, df):
         CS = storage.Client()
-        BUCKET_NAME = 'scraper_data'
-        BLOB_NAME = '{utility}_historical_data.csv'.format(
-            utility=self.utility)
+        dateString = datetime.today().strftime('%Y-%m-%d')
+        BUCKET_NAME = 'scraper_data_' + stage
+        BLOB_NAME = '{utility}_historical_data_{date}.csv'.format(
+            utility=self.utility,
+            date=dateString
+        )
 
         """Uploads a file to the bucket."""
         bucket = CS.get_bucket(BUCKET_NAME)

cloud_functions/serverless.yml

@@ -6,6 +6,8 @@ provider:
   runtime: python37
   region: us-central1
   project: ${opt:id, 'japan-grid-carbon-api'}
+  environment:
+    STAGE: ${opt:stage, self:provider.stage, 'production'}
   # The GCF credentials can be a little tricky to set up. Luckily we've documented this for you here:
   # https://serverless.com/framework/docs/providers/google/guide/credentials/
   #

local.sh

@@ -1,11 +1,28 @@
 #!/bin/bash
+yell() { echo "$0: $*" >&2; }
+die() { yell "$*"; exit 111; }
+try() { "$@" || die "cannot $*"; }
+
 FUNCTION=$1
+ENV=$2
 DIR="$PWD"
-export GCP_PROJECT="japan-grid-carbon-api"
 
-export GOOGLE_APPLICATION_CREDENTIALS="$DIR/.gcloud/keyfile.json"
 
-echo "Locally Running Function $FUNCTION"
+if [ $ENV == 'production' ]
+then
+    export GCP_PROJECT="japan-grid-carbon-api"
+elif [ $ENV == 'staging' ]
+then
+    export GCP_PROJECT="japan-grid-carbon-api-staging"
+else
+    echo "Not a valid environment - $ENV"
+    exit 1
+fi
+
+export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/.gcloud/japan-grid-carbon-service-key-$ENV.json"
+export STAGE=$ENV
+
+echo "Locally Running Function $FUNCTION in $ENV"
 cd cloud_functions/
 
 functions-framework --target $FUNCTION --debug