From f5cb846b9272819f19652a39c1bd26725089c75b Mon Sep 17 00:00:00 2001 From: Shankari Date: Mon, 17 Apr 2023 15:57:44 -0700 Subject: [PATCH 1/3] Don't build images for the `join_redirect_to_static` branch That was a hacky branch which we are now ready to remove We have removed the branch, we are now removing the automatic build from the branch This is a partial fix for https://github.com/e-mission/e-mission-docs/issues/855 --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index a14595aad..3b991786a 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,7 +6,7 @@ name: docker image # events but only for the master branch on: push: - branches: [ master, gis-based-mode-detection, join_redirect_to_static ] + branches: [ master, gis-based-mode-detection ] # Env variable From 4345b269be74555c7d1ec3e5f1f1e269efd96839 Mon Sep 17 00:00:00 2001 From: Shankari Date: Mon, 17 Apr 2023 17:55:51 -0700 Subject: [PATCH 2/3] :fire: Remove unused auth methods We currently only support token-based authentication, either using a token generated on the phone (`skip`) or pre-generated (`token_list`). Removing the google auth and openid authentication methods, and the python package used for the auth to slim down our dependency profile Testing done: - Launched server with changes - Logged in via the UI --- emission/net/auth/auth.py | 8 ---- emission/net/auth/google_auth.py | 71 -------------------------------- emission/net/auth/openid_auth.py | 64 ---------------------------- setup/environment36.yml | 1 - 4 files changed, 144 deletions(-) delete mode 100644 emission/net/auth/google_auth.py delete mode 100644 emission/net/auth/openid_auth.py diff --git a/emission/net/auth/auth.py b/emission/net/auth/auth.py index 6ad2ce758..561940600 100644 --- a/emission/net/auth/auth.py +++ b/emission/net/auth/auth.py @@ -20,14 +20,6 @@ def getAuthMethod(methodName): import emission.net.auth.secret as enar logging.debug("methodName = secret, returning %s" % enar.SecretMethod) return enar.SecretMethod() - elif methodName == "openid_auth": - import emission.net.auth.openid_auth as enao - logging.debug("methodName = openid_auth, returning %s" % enao.OpenIDAuthMethod) - return enao.OpenIDAuthMethod() - elif methodName == "google_auth": - import emission.net.auth.google_auth as enag - logging.debug("methodName = google_auth, returning %s" % enag.GoogleAuthMethod) - return enag.GoogleAuthMethod() elif methodName == "token_list": import emission.net.auth.token_list as enat logging.debug("methodName = token_list, returning %s" % enat.TokenListMethod) diff --git a/emission/net/auth/google_auth.py b/emission/net/auth/google_auth.py deleted file mode 100644 index f3171accd..000000000 --- a/emission/net/auth/google_auth.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() -from builtins import * -from builtins import object -import logging -import json -import traceback -import requests - -# For decoding JWTs on the client side -import google.oauth2.id_token as goi -import google.auth.transport.requests as gatr - -class GoogleAuthMethod(object): - def __init__(self): - key_file = open('conf/net/auth/google_auth.json') - key_data = json.load(key_file) - key_file.close() - self.client_key = key_data["client_key"] - self.client_key_old = key_data["client_key_old"] - self.ios_client_key = key_data["ios_client_key"] - self.ios_client_key_new = key_data["ios_client_key_new"] - self.valid_keys = [self.client_key, self.client_key_old, - self.ios_client_key, self.ios_client_key_new] - - # Code snippet from - # https://developers.google.com/identity/sign-in/android/backend-auth - def __verifyTokenFields(self, tokenFields, audienceKey, issKey): - if audienceKey not in tokenFields: - raise ValueError("Invalid token %s, does not contain %s" % - (tokenFields, audienceKey)) - in_client_key = tokenFields[audienceKey] - if in_client_key not in self.valid_keys: - raise ValueError("Incoming client key %s not in valid list %s" % - (in_client_key, self.valid_keys)) - - if issKey not in tokenFields: - raise ValueError("Invalid token %s" % tokenFields) - - in_issuer = tokenFields[issKey] - issuer_valid_list = ['accounts.google.com', 'https://accounts.google.com'] - if in_issuer not in issuer_valid_list: - raise ValueError('Wrong issuer %s, expected %s' % (in_issuer, issuer_valid_list)) - - return tokenFields['email'] - - - def verifyUserToken(self, token): - try: - # attempt to validate token on the client-side - logging.debug("Using the google auth library to verify id token of length %d from android phones" % len(token)) - tokenFields = goi.verify_oauth2_token(token, gatr.Request()) - logging.debug("tokenFields from library = %s" % tokenFields) - verifiedEmail = self.__verifyTokenFields(tokenFields, "aud", "iss") - logging.debug("Found user email %s" % tokenFields['email']) - return verifiedEmail - except: - logging.debug("OAuth failed to verify id token, falling back to constructedURL") - #fallback to verifying using Google API - constructedURL = ("https://www.googleapis.com/oauth2/v1/tokeninfo?id_token=%s" % token) - r = requests.get(constructedURL) - tokenFields = json.loads(r.content) - logging.debug("tokenFields from constructedURL= %s" % tokenFields) - verifiedEmail = self.__verifyTokenFields(tokenFields, "audience", "issuer") - logging.debug("Found user email %s" % tokenFields['email']) - return verifiedEmail - diff --git a/emission/net/auth/openid_auth.py b/emission/net/auth/openid_auth.py deleted file mode 100644 index 8ca7b7719..000000000 --- a/emission/net/auth/openid_auth.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() -from builtins import * -from builtins import object -import urllib.request, urllib.parse, urllib.error, json -import json -import logging -# Disable this due to https://security.snyk.io/vuln/SNYK-PYTHON-JWCRYPTO-3030035 -# We should consider removing this completely when we remove the openid option from the app as well -# from jwcrypto import jwt, jwk - - -class OpenIDAuthMethod(object): - def __init__(self): - """ - Retrieve auth server config and set up the validator - - :param config_url: the discovery URI - :param audience: client ID to verify against - """ - # first read the parameters from the keys.json file - # including the config_url and the audience - key_file = open('conf/net/auth/openid_auth.json') - key_data = json.load(key_file) - key_file.close() - discoveryURI = key_data["discoveryURI"] - audience = key_data["clientID"] - # Now, use them to retrieve the configuration - self.config = json.loads(OpenIDAuthMethod.__fetch_content__(discoveryURI)) - self.config['audience'] = audience - - # Fetch signing key/certificate - jwk_response = OpenIDAuthMethod.__fetch_content__(self.config['jwks_uri']) - self.jwk_keyset = jwk.JWKSet.from_json(jwk_response) - - @staticmethod - def __fetch_content__(url): - response = urllib.request.urlopen(url) - return response.read() - - def __verify_claim__(self, decoded_token_json): - if decoded_token_json['iss'] != self.config['issuer']: - raise Exception('Invalid Issuer') - if decoded_token_json['aud'] != self.config['audience']: - raise Exception('Invalid Audience') - - def verifyUserToken(self, token): - """ - Verify the token with the provided JWK certificate and claims - - :param token: the token to verify - :return: the decoded ID token body - """ - decoded_token = jwt.JWT(key=self.jwk_keyset, jwt=token) - decoded_json = json.loads(decoded_token.claims) - logging.debug("decoded_json = %s" % decoded_json) - self.__verify_claim__(decoded_json) - email = decoded_json['email'] - logging.debug("After verifying claim, returning valid email = %s" % email) - return email diff --git a/setup/environment36.yml b/setup/environment36.yml index 4b3d4b45d..66a5dcc89 100644 --- a/setup/environment36.yml +++ b/setup/environment36.yml @@ -8,7 +8,6 @@ dependencies: - cheroot=9.0.0 - future=0.18.3 - geojson=2.5.0 -- google-auth=2.16.0 - jsonpickle=3.0.0 - numpy=1.24.2 - pandas=1.5.3 From c7aae9bfc57a4a4febe223fdc79952e631321da2 Mon Sep 17 00:00:00 2001 From: Shankari Date: Mon, 17 Apr 2023 21:07:23 -0700 Subject: [PATCH 3/3] Remove previously commented out dependencies, comment out more and switch to jq - Previously commented out dependencies were "deprecated"; remove them - Comment out other lines that don't seem to be needed - e.g. editors; these are now deprecated and will be removed in the next edit - switch to jq instead of sed to support more complex mongoDB URLs and be more consistent with the internal server build - install wget in addition to curl so we don't need to use `apt-get update` in the internal repo; we should be able to remove it in the next iteration if we are not using it. Testing done: - Launched the container as part of a docker-compose - Checked the logs and the server started up - Tried to connect to the server using a browser, was redirected to the notfound URL - Tried to connect to the server using the phone app, got a 403 error (since we hadn't loaded any data) --- .docker/docker_start_script.sh | 6 +++--- Dockerfile | 25 +++++++++++-------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.docker/docker_start_script.sh b/.docker/docker_start_script.sh index c66c17727..f41528c0f 100644 --- a/.docker/docker_start_script.sh +++ b/.docker/docker_start_script.sh @@ -7,9 +7,9 @@ echo ${DB_HOST} if [ -z ${DB_HOST} ] ; then local_host=`hostname -i` - sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf else - sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf fi cat conf/storage/db.conf @@ -38,4 +38,4 @@ fi source setup/activate.sh # launch the webapp -./e-mission-py.bash emission/net/api/cfc_webapp.py \ No newline at end of file +./e-mission-py.bash emission/net/api/cfc_webapp.py diff --git a/Dockerfile b/Dockerfile index c8523e803..f1a259ae1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,21 @@ # python 3 -FROM ubuntu:focal +FROM ubuntu:jammy MAINTAINER K. Shankari (shankari@eecs.berkeley.edu) WORKDIR /usr/src/app -RUN apt-get update -RUN apt-get install -y curl -RUN apt-get install -y git +RUN apt-get -y -qq update +RUN apt-get install -y -qq curl +RUN apt-get install -y -qq wget +# RUN apt-get install -y git # install nano and vim for editing -RUN apt-get -y install nano vim +# RUN apt-get -y install nano vim + +# install jq to parse json within bash scripts +RUN curl -o /usr/local/bin/jq http://stedolan.github.io/jq/download/linux64/jq && \ + chmod +x /usr/local/bin/jq # cleanup RUN apt-get -y remove --purge build-essential @@ -18,15 +23,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY . . -# ARG SERVER_REPO -# ENV SERVER_REPO=${SERVER_REPO:-https://github.com/e-mission/e-mission-server.git} - -# ARG SERVER_BRANCH -# ENV SERVER_BRANCH=${SERVER_BRANCH:-master} - -# ADD clone_server.sh /clone_server.sh RUN chmod u+x ./.docker/setup_config.sh -# ADD index.html /index.html # # This clone puts the server code into the image, not the container RUN bash -c "./.docker/setup_config.sh" @@ -41,4 +38,4 @@ RUN chmod u+x ./.docker/docker_start_script.sh EXPOSE 8080 -CMD ["/bin/bash", "./.docker/docker_start_script.sh"] \ No newline at end of file +CMD ["/bin/bash", "./.docker/docker_start_script.sh"]