Fix build issues

.dockerignore

@@ -22,3 +22,4 @@ db-data
 workers-db-data
 cluster-db-data
 ./workers/cluster-db-data
+

Dockerfile.base

@@ -10,7 +10,7 @@ ADD ./requirements.txt /code/requirements.txt
 ADD ./pytest.ini /code/pytest.ini
 
 # Install dependencies
-RUN conda install -c conda-forge "python>=3.7.0" pip "paramtools>=0.5.4" "bokeh==1.2.0" gcsfs --yes && \
+RUN conda install -c conda-forge "python>=3.7.0" pip "paramtools>=0.5.4" gcsfs --yes && \
     pip install -r requirements.txt
 
 # Add our code

Dockerfile.dev

@@ -1,7 +1,9 @@
-FROM webbase
+FROM webbase:latest
 
-RUN conda install -c conda-forge pylint black --yes
+# RUN conda install -c conda-forge pylint black --yes
 
+COPY ./google-creds.json ./google-creds.json
 ENV GOOGLE_APPLICATION_CREDENTIALS ./google-creds.json
 
+WORKDIR /code
 CMD python manage.py runserver 0.0.0.0:$PORT

db-secret.sh

@@ -0,0 +1,7 @@
+kubectl create secret generic web-db-secret \
+  --from-literal=username=postgres \
+  --from-literal=password=$(cs secrets get HDOUPE_POSTGRES_PASSWORD) \
+  --from-literal=database=postgres \
+  --from-literal=host=db \
+  --namespace web
+

deploy/cs_deploy/webapp.py

@@ -147,11 +147,12 @@ def write_web(self, dev=False):
             "DATABASE_URL",
             "DEFAULT_CLUSTER_USER",
             "DEFAULT_VIZ_HOST",
-            "USE_STRIPE",
         ]:
             if webapp_config.get(var, None):
                 web_configmap["data"][var] = webapp_config[var]
 
+        web_configmap["data"]["USE_STRIPE"] = False
+
         if dev:
             warnings.warn("Web deployment is being created in DEBUG mode!")
             spec["containers"][0]["volumeMounts"] = [

local-deployment.md

@@ -60,6 +60,8 @@
    cs secrets set DJANGO_SECRET_KEY $(openssl rand -hex 32)
    cs secrets set WEB_CS_CRYPT_KEY $(openssl rand -hex 32)
    cs secrets set CS_CRYPT_KEY $(openssl rand -hex 32)
+   # Swap HDOUPE out for your username of choice :)
+   cs secrets set HDOUPE_POSTGRES_PASSWORD $(openssl rand -hex 32)
    ```
 
 1. Build webapp docker image:
@@ -220,3 +222,28 @@ kubectl delete pods $DB_POD
 export WEB_POD=$(kubectl get pod -l app=web -o jsonpath="{.items[0].metadata.name}")
 kubectl delete pods $WEB_POD
 ```
+
+### Workers
+
+```
+cs secrets set WORKERS_DB_PASSWORD $(openssl rand -hex 32)
+# Swap HDOUPE out for your username of choice :)
+cs secrets set HDOUPE_WORKERS_REDIS_PASSWORD $(openssl rand -hex 32)
+cs secrets set HDOUPE_WORKERS_API_SECRET_KEY $(openssl rand -hex 32)
+```
+
+```
+helm template cs-workers \
+    --set project=$PROJECT \
+    --set tag=$TAG \
+    --set api.secret_key=$(cs secrets get HDOUPE_WORKERS_API_SECRET_KEY) \
+    --set redis.password=$(cs secrets get HDOUPE_WORKERS_REDIS_PASSWORD) \
+    --set db.password=$(cs secrets get WORKERS_DB_PASSWORD)  \
+    --namespace workers
+```
+
+```
+kubectl exec --namespace workers -it deployments/api -- /bin/bash
+cd api
+alembic upgrade head
+```

local_web_dev.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# kubectl config use-context kind-cs &&
+
+    source env.sh && \
+    export TAG=$1 && \
+    kubectl create namespace web && \
+    kubectl create secret generic web-db-secret \
+        --from-literal=username=postgres \
+        --from-literal=password=$(cs secrets get HDOUPE_POSTGRES_PASSWORD) \
+        --from-literal=database=postgres \
+        --from-literal=host=db \
+        --namespace web && \
+
+
+    cs webapp build --dev && \
+    cs webapp push --use-kind && \
+    cs webapp config -o - --update-db --dev | kubectl apply --namespace web -f -
+

requirements.txt

@@ -1,4 +1,5 @@
 django>=3.2,<4
+jinja2
 djangorestframework
 msgpack
 requests>=2.20
@@ -12,13 +13,13 @@ pytest
 pytest-django
 django-widget-tweaks
 django-crispy-forms
-psycopg2-binary==2.8.6
+psycopg2-binary
 ipython
 markdown
 django-guardian
 hashids
 django-rest-auth
-django-allauth
+django-allauth==0.54.0
 cs-storage>=1.11.0
 cs-crypt>=0.0.2
 pyjwt

web-kubernetes/deployment-cleanup.template.yaml

@@ -1,4 +1,4 @@
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: web-deployment-cleanup

webapp/apps/billing/events.py

@@ -5,7 +5,7 @@
 from .models import Customer
 
 
-stripe.api_key = os.environ.get('STRIPE_SECRET')
+# stripe.api_key = os.environ.get('STRIPE_SECRET')
 
 
 def get_customer_from_event(stripe_event):

webapp/apps/billing/invoice.py

@@ -19,7 +19,7 @@
 
 import stripe
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 def process_simulations(simulations):

webapp/apps/billing/models.py

@@ -17,7 +17,7 @@
 
 from .email import send_subscribe_to_plan_email, send_sub_canceled_email
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 def timestamp_to_datetime(timestamp):

webapp/apps/billing/tests/test_models.py

@@ -22,7 +22,7 @@
 from webapp.apps.billing.utils import create_three_month_pro_subscription
 
 User = get_user_model()
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 def time_is_close(a, b):

webapp/apps/billing/utils.py

@@ -6,7 +6,7 @@
 
 import stripe
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 def update_payment(user, stripe_token):

webapp/apps/billing/views.py

@@ -25,7 +25,7 @@
 
 from .utils import update_payment, has_payment_method
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 wh_secret = os.environ.get("WEBHOOK_SECRET")
 
 white_listed_urls = set(

webapp/apps/billing/webhooks.py

@@ -10,7 +10,7 @@
 from .events import get_customer_from_event
 
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 def customer_created(event):

webapp/apps/comp/serializers.py

@@ -55,7 +55,7 @@ class MiniSimulationSerializer(serializers.ModelSerializer):
     status = serializers.CharField(required=False)
     api_url = serializers.CharField(required=False, source="get_absolute_api_url")
     gui_url = serializers.CharField(required=False, source="get_absolute_url")
-    is_public = serializers.NullBooleanField(required=False)
+    is_public = serializers.BooleanField(required=False)
 
     # see to_representation
     # role = serializers.BooleanField(source="role")

webapp/apps/comp/views/views.py

@@ -5,7 +5,6 @@
 import os
 import re
 
-from bokeh.resources import CDN
 import requests
 
 from django.db import models

webapp/apps/conftest.py

@@ -35,7 +35,7 @@
 from webapp.apps.comp.models import Inputs, Simulation
 
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# # stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 ###########################User/Billing Fixtures###############################
 from django.conf import settings
@@ -59,13 +59,13 @@ def django_db_setup(django_db_setup, django_db_blocker):
 
         # User for pushing outputs from the workers to the webapp.
         comp_api_user = User.objects.create_user(
-            username="comp-api-user",
-            email="comp-api-user@email.com",
+            username="cs-api-user",
+            email="cs-api-user@email.com",
             password="heyhey2222",
         )
 
-        webapp.settings.DEFAULT_CLUSTER_USER = "comp-api-user"
-        webapp.apps.users.models.DEFAULT_CLUSTER_USER = "comp-api-user"
+        webapp.settings.DEFAULT_CLUSTER_USER = "cs-api-user"
+        webapp.apps.users.models.DEFAULT_CLUSTER_USER = "cs-api-user"
         for u in [modeler, sponsor, hdoupe, comp_api_user]:
             Token.objects.create(user=u)
             Profile.objects.create(user=u, is_active=True)
@@ -385,7 +385,7 @@ class MetaParams(pt.Parameters):
 
 @pytest.fixture
 def comp_api_user(db):
-    return Profile.objects.get(user__username="comp-api-user")
+    return Profile.objects.get(user__username="cs-api-user")
 
 
 ####### Mock requests to kubernetes cluster ##########

webapp/apps/publish/tests/test_views.py

@@ -63,7 +63,7 @@ def test_post(self, client, visibility_params):
         assert project
         assert project.server_cost
 
-        api_user = Profile.objects.get(user__username="comp-api-user")
+        api_user = Profile.objects.get(user__username="cs-api-user")
         assert project.has_write_access(api_user.user)
         assert project.role(api_user.user) == "write"
         assert project.role(project.cluster.service_account.user) == "write"
@@ -389,7 +389,7 @@ def test_tags_api(
         mock_deployments_requests_to_cluster,
     ):
         owner, is_public = visibility_params
-        prof = Profile.objects.get(user__username="comp-api-user")
+        prof = Profile.objects.get(user__username="cs-api-user")
         api_client.force_login(prof.user)
 
         project = test_models[0].project

webapp/apps/publish/urls.py

@@ -1,5 +1,4 @@
-from django.conf.urls import url
-from django.urls import path
+from django.urls import path, re_path
 
 from .views import (
     ProjectView,
@@ -18,7 +17,7 @@
 
 
 urlpatterns = [
-    url(r"^$", ProjectView.as_view(), name="publish"),
+    re_path(r"^$", ProjectView.as_view(), name="publish"),
     path(
         "api/v1/deployments/<str:id>/",
         DeploymentsIdView.as_view(),

webapp/apps/publish/views.py

@@ -218,11 +218,14 @@ class ProjectAPIView(generics.ListAPIView):
         TokenAuthentication,
         ClusterAuthentication,
     )
-    api_user = User.objects.get(username="comp-api-user")
 
     serializer_class = ProjectSerializer
     pagination_class = ProjectResultsPagination
 
+    @property
+    def api_user(self):
+        return User.objects.get(username="cs-api-user")
+
     def get_queryset(self):
         return projects_with_access(self.request.user)
 

webapp/apps/users/forms.py

@@ -13,7 +13,7 @@
 
 User = get_user_model()
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# # stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 
 class UserCreationForm(authforms.UserCreationForm):

webapp/apps/users/tests/utils.py

@@ -15,7 +15,7 @@
 import webapp.apps.users.models
 
 
-stripe.api_key = os.environ.get("STRIPE_SECRET")
+# # stripe.api_key = os.environ.get("STRIPE_SECRET")
 
 User = get_user_model()
 

webapp/settings.py

@@ -164,7 +164,6 @@ def default_db_url():
         "PORT": "5432",
     }
 
-
 DATABASES = {
     "default": default_db_url(),
     # override database name for tests.
@@ -190,7 +189,6 @@ def default_db_url():
 # https://django-allauth.readthedocs.io/en/latest/configuration.html
 ACCOUNT_EMAIL_VERIFICATION = "none"
 ACCOUNT_AUTHENTICATION_METHOD = "username_email"
-ACCOUNT_UNIQUE_EMAIL = False
 
 # Password validation
 # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators

webapp/urls.py

@@ -14,8 +14,7 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, include
-from django.conf.urls import url
+from django.urls import path, include, re_path
 from rest_framework.authtoken import views as rf_views
 
 from webapp.apps.comp import views as compviews
@@ -48,8 +47,8 @@
         compviews.ModelConfigAPIView.as_view(),
         name="modelconfig_api",
     ),
-    url(r"^rest-auth/", include("rest_auth.urls")),
-    url(r"^rest-auth/registration/", include("rest_auth.registration.urls")),
+    re_path(r"^rest-auth/", include("rest_auth.urls")),
+    re_path(r"^rest-auth/registration/", include("rest_auth.registration.urls")),
     path("api/v1/sims", compviews.UserSimsAPIView.as_view(), name="sim_api"),
     path("feed/", pageviews.LogView.as_view(), name="feed"),
     path("log/", pageviews.LogView.as_view(), name="log"),

workers/cs-workers/templates/job-cleanup-Job.yaml

@@ -1,4 +1,4 @@
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: job-cleanup

workers/cs_workers/services/api/database.py

@@ -6,7 +6,7 @@
 from .settings import settings
 
 
-engine = create_engine(settings.SQLALCHEMY_DATABASE_URI, pool_pre_ping=True)
+engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), pool_pre_ping=True)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 Base = declarative_base()