update gusty-demo

airflow/Dockerfile

@@ -1,26 +1,13 @@
-FROM python:3.9.1-slim-buster
+FROM jupyter/r-notebook:2023-03-20
 USER root
 
 # PSQL Requirements
 RUN apt-get update && apt-get install -y libpq-dev build-essential
 
-# Get R
-RUN apt-get -y install libssl-dev libgit2-dev libcurl4-gnutls-dev libxml2-dev openssl
-RUN apt-get -y install dirmngr --install-recommends
-RUN apt-get -y install software-properties-common
-RUN apt-get -y install apt-transport-https
-RUN apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF'
-RUN add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian buster-cran40/'
-RUN apt-get update
-RUN apt-get -y install r-base
-
 # Install R dependencies
-RUN Rscript -e 'install.packages("devtools")'
-RUN Rscript -e 'install.packages("tidyverse")'
-RUN Rscript -e 'install.packages("doParallel")'
-RUN Rscript -e 'install.packages("tidymodels")'
-RUN Rscript -e 'install.packages("ranger")'
-RUN Rscript -e 'install.packages("RPostgres")'
+RUN Rscript -e 'install.packages("ranger", repos = "http://cran.us.r-project.org")'
+RUN Rscript -e 'install.packages("RPostgres", repos = "http://cran.us.r-project.org")'
+RUN Rscript -e 'install.packages("doParallel", repos = "http://cran.us.r-project.org")'
 
 # Python Requirements
 ADD requirements.txt .

airflow/demo-setup/datawarehouse_setup.py

@@ -3,16 +3,18 @@
 import sqlalchemy
 
 uri = PostgresHook.get_connection('postgres_default').get_uri()
-engine = sqlalchemy.create_engine(uri)
+engine = PostgresHook().get_sqlalchemy_engine()
 
 # create database
 if not database_exists(engine.url):
     create_database(engine.url)
-    engine.execute("GRANT ALL PRIVILEGES ON DATABASE {db} TO {user};".format(user = engine.url.username, db = engine.url.database))
+    with engine.begin() as conn:
+        conn.execute("GRANT ALL PRIVILEGES ON DATABASE {db} TO {user};".format(user = engine.url.username, db = engine.url.database))
 
 # create schema, give permissions
 if not engine.dialect.has_schema(engine, 'views'):
-    engine.execute(sqlalchemy.schema.CreateSchema('views'))
-    engine.execute("GRANT ALL PRIVILEGES ON SCHEMA views TO {user};".format(user = engine.url.username))
-    engine.execute("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA views TO {user};".format(user = engine.url.username))
-    engine.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA views GRANT ALL PRIVILEGES ON TABLES TO {user};".format(user = engine.url.username))
+    with engine.begin() as conn:
+        conn.execute(sqlalchemy.schema.CreateSchema('views'))
+        conn.execute("GRANT ALL PRIVILEGES ON SCHEMA views TO {user};".format(user = engine.url.username))
+        conn.execute("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA views TO {user};".format(user = engine.url.username))
+        conn.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA views GRANT ALL PRIVILEGES ON TABLES TO {user};".format(user = engine.url.username))

airflow/requirements.txt

@@ -1,8 +1,8 @@
-apache-airflow==2.4.1
-apache-airflow-providers-postgres==1.0.2
-gusty==0.6.0
-ipykernel==5.5.5
-nbconvert==6.5.1
-siuba==0.0.24
-SQLAlchemy-Utils==0.36.7
-yfinance==0.1.55
+apache-airflow==2.5.2
+apache-airflow-providers-postgres==5.4.0
+gusty==0.18.1
+ipykernel==6.22.0
+nbconvert==7.2.10
+siuba==0.4.2
+SQLAlchemy-Utils==0.40.0
+yfinance==0.2.13

docker-compose.yml

@@ -38,7 +38,7 @@ services:
                   --lastname User \
                   --role Admin
 
-                # Set up datalake db & views schema
+                # Set up warehouse db & views schema
             	  python3 demo-setup/datawarehouse_setup.py
 
                 # Start airflow