diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index db975d44..eadbacfd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -47,8 +47,13 @@ jobs: run: | make -C main dev + - name: Copy SSH private key + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + run: echo $SSH_PRIVATE_KEY | base64 --decode > ./ssh_pvt_key + - name: Build Kubernetes JupyterHub Image - run: make -C main build-hubs-k8 + run: make -C main build-hubs-k8 SSH_PRIVATE_KEY=$(cat ./ssh_pvt_key) env: # Full logs for CI build BUILDKIT_PROGRESS: plain diff --git a/Makefile b/Makefile index ded1ef62..957719fa 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ build-grader-setup-service: ## build grader-setup-service docker image build-hubs-k8: ## build jupyterhub images kubernetes setups @docker build --build-arg BASE_IMAGE=${JUPYTERHUB_K8_BASE_IMAGE}:${JUPYTERHUB_K8_BASE_TAG} -t ${OWNER}/k8s-hub:base-${JUPYTERHUB_K8_BASE_TAG} src/illumidesk/. --no-cache - @docker build --build-arg BASE_IMAGE=${OWNER}/k8s-hub:base-${JUPYTERHUB_K8_BASE_TAG} -t ${OWNER}/k8s-hub:${JUPYTERHUB_K8_BASE_TAG} src/illumideskdummyauthenticator/. --no-cache + @docker build --build-arg BASE_IMAGE=${OWNER}/k8s-hub:base-${JUPYTERHUB_K8_BASE_TAG} --build-arg SSH_PRIVATE_KEY="$(SSH_PRIVATE_KEY)" -t ${OWNER}/k8s-hub:${JUPYTERHUB_K8_BASE_TAG} src/illumideskdummyauthenticator/. --no-cache build-hubs: ## build jupyterhub images for standard docker-compose and docker run setups @docker build --build-arg BASE_IMAGE=${JUPYTERHUB_DOCKER_BASE_IMAGE} -t ${OWNER}/jupyterhub:base-${JUPYTERHUB_DOCKER_BASE_TAG} src/illumidesk/. --no-cache diff --git a/src/async_nbgrader/dev-requirements.txt b/src/async_nbgrader/dev-requirements.txt index 220e1565..2bfe8cb7 100644 --- a/src/async_nbgrader/dev-requirements.txt +++ b/src/async_nbgrader/dev-requirements.txt @@ -15,4 +15,4 @@ sphinx_rtd_theme sphinx-autodoc-typehints nbval requests-mock -wheel +wheel \ No newline at end of file diff --git a/src/formgradernext/requirements.txt b/src/formgradernext/requirements.txt index 5fce9ae0..143fe0b3 100644 --- a/src/formgradernext/requirements.txt +++ b/src/formgradernext/requirements.txt @@ -58,7 +58,7 @@ ipywidgets==7.6.5 # via jupyter jedi==0.18.0 # via ipython -jinja2==3.0.2 +jinja2==3.0.3 # via # nbconvert # notebook @@ -212,6 +212,7 @@ webencodings==0.5.1 # via bleach widgetsnbextension==3.5.2 # via ipywidgets +secretsmanager-illumidesk==0.0.3 # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file diff --git a/src/graderservice/graderservice/graderservice.py b/src/graderservice/graderservice/graderservice.py index 5981cabf..941abd14 100644 --- a/src/graderservice/graderservice/graderservice.py +++ b/src/graderservice/graderservice/graderservice.py @@ -6,10 +6,12 @@ from os import path from pathlib import Path from secrets import token_hex - from kubernetes import client from kubernetes import config from kubernetes.config import ConfigException +from kubernetes.client.rest import ApiException +from secretsmanager.secretsmanager import SecretsManager +import time from .templates import NBGRADER_COURSE_CONFIG_TEMPLATE from .templates import NBGRADER_HOME_CONFIG_TEMPLATE @@ -35,9 +37,6 @@ "ILLUMIDESK_NB_EXCHANGE_MNT_ROOT", "/illumidesk-nb-exchange" ) GRADER_PVC = os.environ.get("GRADER_PVC", "grader-setup-pvc") -GRADER_EXCHANGE_SHARED_PVC = os.environ.get( - "GRADER_SHARED_PVC", "exchange-shared-volume" -) # user UI and GID to use within the grader container NB_UID = os.environ.get("NB_UID", 10001) @@ -53,13 +52,20 @@ JUPYTERHUB_API_URL = os.environ.get("JUPYTERHUB_API_URL") or "http://hub:8081/hub/api" JUPYTERHUB_BASE_URL = os.environ.get("JUPYTERHUB_BASE_URL") or "/" +CAMPUS_ID = os.environ.get("CAMPUS_ID") + # NBGrader database settings to save in nbgrader_config.py file nbgrader_db_host = os.environ.get("POSTGRES_NBGRADER_HOST") nbgrader_db_password = os.environ.get("POSTGRES_NBGRADER_PASSWORD") nbgrader_db_user = os.environ.get("POSTGRES_NBGRADER_USER") nbgrader_db_port = os.environ.get("POSTGRES_NBGRADER_PORT") -nbgrader_db_name = os.environ.get("POSTGRES_NBGRADER_DB_NAME") +nbgrader_db_name = os.environ.get("POSTGRES_NBGRADER_DATABASE") or "illumidesk" +aws_secret_arn = os.environ.get('AWS_SECRET_ARN') +region = os.environ.get('AWS_REGION') or 'us-west-2' +secretmanager = SecretsManager(aws_secret_arn, region_name=region) +if secretmanager.host == "": + secretmanager.host = nbgrader_db_host class GraderServiceLauncher: def __init__(self, org_name: str, course_id: str): @@ -180,11 +186,16 @@ def _create_nbgrader_files(self): logger.info( f"Writing the nbgrader_config.py file at jupyter directory (within the grader home): {grader_nbconfig_path}" ) + db_url = '' + if aws_secret_arn: + db_url = secretmanager.rds_connection(nbgrader_db_name) + else: + db_url = f"postgresql://{nbgrader_db_user}:{nbgrader_db_password}@{nbgrader_db_host}:5432/{nbgrader_db_name}" # write the file grader_home_nbconfig_content = NBGRADER_HOME_CONFIG_TEMPLATE.format( grader_name=self.grader_name, course_id=self.course_id, - db_url=f"postgresql://{nbgrader_db_user}:{nbgrader_db_password}@{nbgrader_db_host}:5432/{self.org_name}_{self.course_id}", + db_url=db_url, ) grader_nbconfig_path.write_text(grader_home_nbconfig_content) # Write the nbgrader_config.py file at grader home directory @@ -269,6 +280,12 @@ def _create_deployment_object(self): client.V1EnvVar(name="NB_UID", value=str(NB_UID)), client.V1EnvVar(name="NB_GID", value=str(NB_GID)), client.V1EnvVar(name="NB_USER", value=self.grader_name), + client.V1EnvVar(name="CAMPUS_ID", value=str(CAMPUS_ID)), + client.V1EnvVar(name="POSTGRES_JUPYTERHUB_HOST", value=str(nbgrader_db_host)), + client.V1EnvVar(name="POSTGRES_JUPYTERHUB_USER", value=str(nbgrader_db_user)), + client.V1EnvVar(name="POSTGRES_JUPYTERHUB_PASSWORD", value=str(nbgrader_db_password)), + client.V1EnvVar(name="POSTGRES_JUPYTERHUB_PORT", value=str(nbgrader_db_port)), + client.V1EnvVar(name="POSTGRES_JUPYTERHUB_DB", value=str(nbgrader_db_name)), ], volume_mounts=[ client.V1VolumeMount( @@ -278,7 +295,7 @@ def _create_deployment_object(self): ), client.V1VolumeMount( mount_path="/srv/nbgrader/exchange", - name=GRADER_EXCHANGE_SHARED_PVC, + name=GRADER_PVC, sub_path=sub_path_exchange, ), ], @@ -298,12 +315,6 @@ def _create_deployment_object(self): claim_name=GRADER_PVC ), ), - client.V1Volume( - name=GRADER_EXCHANGE_SHARED_PVC, - persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource( - claim_name=GRADER_EXCHANGE_SHARED_PVC - ), - ), ], ), ) @@ -358,3 +369,35 @@ def update_jhub_deployment(self): name="hub", namespace=NAMESPACE, body=deployment ) logger.info(f"Jhub patch response:{api_response}") + + # Restarts deployment in namespace + def restart_deployment(self, deployment, namespace): + now = datetime.utcnow() + now = str(now.isoformat("T") + "Z") + body = { + 'spec': { + 'template': { + 'metadata': { + 'annotations': { + 'kubectl.kubernetes.io/restartedAt': now + } + } + } + } + } + deployment_status = f'{deployment} failed to deploy to organization: {namespace}', 404 + try: + restart_deployment = self.apps_v1.patch_namespaced_deployment(deployment, namespace, body, pretty='true') + except ApiException as e: + logger.error("Exception when calling AppsV1Api->read_namespaced_deployment_status: %s\n" % e) + except Exception as e: + logger.error(deployment_status, e) + else: + while restart_deployment.status.updated_replicas != restart_deployment.spec.replicas: + logger.info(f'Waiting for status to update for grader{deployment} to organization {namespace}') + time.sleep(5) + deployment_status = f'{deployment} successfully deployed to organization {namespace}', 200 + return deployment_status + + + diff --git a/src/graderservice/graderservice/routes.py b/src/graderservice/graderservice/routes.py index 93416c4d..4a9fc3cc 100644 --- a/src/graderservice/graderservice/routes.py +++ b/src/graderservice/graderservice/routes.py @@ -203,7 +203,23 @@ def assignment_dir_creation(org_name: str, course_id: str, assignment_name: str) success=True, message=f"Created new assignment directory: {assignment_dir}", ) +@grader_setup_bp.route( + "/services///restart", methods=["POST"] +) +def restart_grader(org_name: str, course_id: str): + launcher = GraderServiceLauncher(org_name=org_name, course_id=course_id) + try: + restart_deployment_status = launcher.restart_deployment(f'grader-{course_id}',org_name) + except Exception as e: + logger.error(f"Error restarting grader: {e}") + logger.info(restart_deployment_status) + success = True if restart_deployment_status[1]==200 else False + return jsonify( + success=success, + message=f"{restart_deployment_status[0]}" + ), restart_deployment_status[1] + @grader_setup_bp.route("/healthcheck") def healthcheck(): diff --git a/src/graderservice/requirements.txt b/src/graderservice/requirements.txt index aea7f9e2..c22fcb1f 100644 --- a/src/graderservice/requirements.txt +++ b/src/graderservice/requirements.txt @@ -30,9 +30,9 @@ idna==2.10 # via requests itsdangerous==2.0.0 # via flask -jinja2==3.0.0 +jinja2==3.0.3 # via flask -kubernetes==12.0.1 +kubernetes==17.17.0 # via graderservice (src/graderservice/setup.py) markupsafe==2.0.0 # via jinja2 @@ -70,8 +70,9 @@ urllib3==1.26.5 # requests websocket-client==0.59.0 # via kubernetes -werkzeug==2.0.0 +werkzeug==2.0.2 # via flask +secretsmanager-illumidesk==0.0.3 # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file diff --git a/src/graderservice/setup.py b/src/graderservice/setup.py index 61c30550..174156cf 100644 --- a/src/graderservice/setup.py +++ b/src/graderservice/setup.py @@ -37,7 +37,7 @@ "flask==1.1.2", "flask-sqlalchemy==2.5.1", "gunicorn==20.0.4", - "kubernetes==12.0.1", + "kubernetes==17.17.0", ], # noqa: E231 package_data={ "": ["*.html"], diff --git a/src/illumidesk/Dockerfile b/src/illumidesk/Dockerfile index 15fb9d67..d94f7d34 100644 --- a/src/illumidesk/Dockerfile +++ b/src/illumidesk/Dockerfile @@ -1,37 +1,79 @@ # for kubernetes, use the --build-arg when building image or uncomment -# ARG BASE_IMAGE=jupyterhub/k8s-hub:1.1.2 -ARG BASE_IMAGE=jupyterhub/jupyterhub:1.4.2 +ARG BASE_IMAGE=jupyterhub/k8s-hub:1.1.2 +#ARG BASE_IMAGE=jupyterhub/jupyterhub:1.4.2 +ARG SSH_PRIVATE_KEY FROM "${BASE_IMAGE}" USER root + RUN apt-get update \ && apt-get install -y \ curl \ git \ unzip \ wget \ + openssh-server \ + libmysqlclient-dev \ && rm -rf /var/lib/apt/lists/* +USER "${NB_USER}" + +ARG SSH_PRIVATE_KEY +RUN mkdir ~/.ssh/ +RUN echo "${SSH_PRIVATE_KEY}" > ~/.ssh/id_ed25519 +RUN chmod 600 ~/.ssh/id_ed25519 +RUN ssh-keyscan github.com >> ~/.ssh/known_hosts +# Print SSH_PRIVATE_KEY (for test) +RUN echo "${SSH_PRIVATE_KEY}" + +# # Authorize SSH Host +# RUN mkdir -p "/home/${NB_USER}/.ssh" && \ +# chmod 0700 "/home/${NB_USER}/.ssh" + +# COPY ./id_illumidesk_ssh "/home/${NB_USER}/.ssh/id_rsa" +# COPY ./id_illumidesk_ssh.pub "/home/${NB_USER}/.ssh/id_rsa.pub" + +# # Add the keys and set permissions +# RUN chmod 600 "/home/${NB_USER}/.ssh/id_rsa" && \ +# chmod 600 "/home/${NB_USER}/.ssh/id_rsa.pub" && \ +# touch "/home/${NB_USER}/.ssh"/known_hosts + +# RUN chown -R "${NB_USER}" "/home/${NB_USER}/.ssh" + +# # RUN ssh-keyscan github.com >> "/home/${NB_USER}/.ssh"/known_hosts +# RUN ssh-keyscan -t ssh-ed25519 github.com >> "/home/${NB_USER}/.ssh"/known_hosts + +# RUN file="/home/${NB_USER}/.ssh"/known_hosts && echo $file + +# # RUN echo "Host github.com\n\tStrictHostKeyChecking no\n" >> "/home/${NB_USER}/.ssh/config" +# RUN file="/home/${NB_USER}/.ssh/config" && echo $file + WORKDIR /tmp RUN wget https://configs.illumidesk.com/images/illumidesk-80.png \ && cp -r /tmp/illumidesk-80.png /srv/jupyterhub/ \ && cp -r /tmp/illumidesk-80.png /usr/local/share/jupyterhub/static/images/illumidesk-80.png \ && chown "${NB_UID}" /srv/jupyterhub/illumidesk-80.png -USER "${NB_UID}" +RUN wget https://raw.githubusercontent.com/jupyterhub/jupyterhub/main/examples/service-announcement/announcement.py \ + && mkdir -p /etc/jupyterhub-services/ \ + && cp -r /tmp/announcement.py /etc/jupyterhub-services/ \ + && chown "${NB_UID}" /etc/jupyterhub-services/ ENV PATH="/home/${NB_USER}/.local/bin:${PATH}" +# ENV PYTHONUNBUFFERED 1 # ensure pip is up to date RUN python3 -m pip install --upgrade pip COPY requirements.txt /tmp/ -RUN pip install -r /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt --use-deprecated=legacy-resolver WORKDIR /tmp COPY . /tmp RUN python3 -m pip install /tmp/. +RUN rm -rf "/home/${NB_USER}/.ssh/" + WORKDIR /srv/jupyterhub/ # This config is overwitten with k8s setup diff --git a/src/illumidesk/illumidesk/apis/nbgrader_service.py b/src/illumidesk/illumidesk/apis/nbgrader_service.py index d9e1cae7..1497d8bf 100644 --- a/src/illumidesk/illumidesk/apis/nbgrader_service.py +++ b/src/illumidesk/illumidesk/apis/nbgrader_service.py @@ -9,6 +9,8 @@ from sqlalchemy_utils import database_exists from illumidesk.authenticators.utils import LTIUtils +from secretsmanager.secretsmanager import SecretsManager + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -18,6 +20,7 @@ nbgrader_db_port = os.environ.get("POSTGRES_NBGRADER_PORT") or 5432 nbgrader_db_password = os.environ.get("POSTGRES_NBGRADER_PASSWORD") nbgrader_db_user = os.environ.get("POSTGRES_NBGRADER_USER") +nbgrader_db_name = os.environ.get("POSTGRES_NBGRADER_DATABASE") mnt_root = os.environ.get("ILLUMIDESK_MNT_ROOT", "/illumidesk-courses") org_name = os.environ.get("ORGANIZATION_NAME") or "my-org" @@ -25,16 +28,33 @@ if not org_name: raise EnvironmentError("ORGANIZATION_NAME env-var is not set") +CAMPUS_ID = os.environ.get("CAMPUS_ID") +if not CAMPUS_ID: + raise EnvironmentError("CAMPUS_ID env-var is not set") + +aws_secret_arn = os.environ.get('AWS_SECRET_ARN') +region = os.environ.get('AWS_REGION') or 'us-west-2' +secretmanager = SecretsManager(aws_secret_arn, region_name=region) +if secretmanager.host == "": + secretmanager.host = nbgrader_db_host -def nbgrader_format_db_url(course_id: str) -> str: + +def nbgrader_format_db_url() -> str: + """ + Returns the nbgrader database url """ - Returns the nbgrader database url with the format: _ - Args: - course_id: the course id (usually associated with the course label) from which the launch was initiated. + if aws_secret_arn: + return secretmanager.rds_connection(nbgrader_db_name) + return f"postgresql://{nbgrader_db_user}:{nbgrader_db_password}@{nbgrader_db_host}:{nbgrader_db_port}/{nbgrader_db_name}" + + + +def jupyter_format_db_url() -> str: + """ + Returns the jupyter database url with the format: campus-id. """ - course_id = LTIUtils().normalize_string(course_id) - database_name = f"{org_name}_{course_id}" + database_name = CAMPUS_ID return f"postgresql://{nbgrader_db_user}:{nbgrader_db_password}@{nbgrader_db_host}:{nbgrader_db_port}/{database_name}" @@ -51,7 +71,7 @@ class NbGraderServiceHelper: database_name: the database name """ - def __init__(self, course_id: str, check_database_exists: bool = False): + def __init__(self, course_id: str): if not course_id: raise ValueError("course_id missing") @@ -62,41 +82,43 @@ def __init__(self, course_id: str, check_database_exists: bool = False): self.uid = int(os.environ.get("NB_GRADER_UID") or "10001") self.gid = int(os.environ.get("NB_GRADER_GID") or "100") - self.db_url = nbgrader_format_db_url(course_id) - self.database_name = f"{org_name}_{self.course_id}" - if check_database_exists: - self.create_database_if_not_exists() + self.db_url = nbgrader_format_db_url() + self.create_jupyterhub_database_if_not_exists() - def create_database_if_not_exists(self) -> None: + def create_jupyterhub_database_if_not_exists(self) -> None: """Creates a new database if it doesn't exist""" - conn_uri = nbgrader_format_db_url(self.course_id) + conn_uri = jupyter_format_db_url() if not database_exists(conn_uri): logger.debug("db not exist, create database") create_database(conn_uri) - def add_user_to_nbgrader_gradebook(self, username: str, lms_user_id: str) -> None: + def add_user_to_nbgrader_gradebook(self, email: str, external_user_id: str, source: str, source_type: str, role_name: str = None) -> None: """ Adds a user to the nbgrader gradebook database for the course. Args: - username: The user's username - lms_user_id: The user's id on the LMS + email: The user's email + external_user_id: The user's id on the external system + source: source from where user was authenticated + source_type: source_type + role_name: role of the user Raises: InvalidEntry: when there was an error adding the user to the database """ - if not username: - raise ValueError("username missing") - if not lms_user_id: - raise ValueError("lms_user_id missing") + if not email: + raise ValueError("email missing") + if not external_user_id: + raise ValueError("external_user_id missing") - with Gradebook(self.db_url, course_id=self.course_id) as gb: + with Gradebook(self.db_url, course_id=self.course_id, campus_id=CAMPUS_ID) as gb: try: - gb.update_or_create_student(username, lms_user_id=lms_user_id) + user = gb.update_or_create_user_by_email(email, role_name=role_name, external_user_id=external_user_id, source=source, source_type=source_type) logger.debug( - "Added user %s with lms_user_id %s to gradebook" - % (username, lms_user_id) + "Added user %s with external_user_id %s to gradebook" + % (email, external_user_id) ) + return user.to_dict() except InvalidEntry as e: logger.debug("Error during adding student to gradebook: %s" % e) @@ -104,14 +126,14 @@ def update_course(self, **kwargs) -> None: """ Updates the course in nbgrader database """ - with Gradebook(self.db_url, course_id=self.course_id) as gb: + with Gradebook(self.db_url, course_id=self.course_id, campus_id=CAMPUS_ID) as gb: gb.update_course(self.course_id, **kwargs) def get_course(self) -> Course: """ Gets the course model instance """ - with Gradebook(self.db_url, course_id=self.course_id) as gb: + with Gradebook(self.db_url, course_id=self.course_id, campus_id=CAMPUS_ID) as gb: course = gb.check_course(self.course_id) logger.debug(f"course got from db:{course}") return course @@ -131,10 +153,10 @@ def register_assignment(self, assignment_name: str, **kwargs: dict) -> Assignmen "Assignment name normalized %s to save in gradebook" % assignment_name ) assignment = None - with Gradebook(self.db_url, course_id=self.course_id) as gb: + with Gradebook(self.db_url, course_id=self.course_id, campus_id=CAMPUS_ID) as gb: try: assignment = gb.update_or_create_assignment(assignment_name, **kwargs) logger.debug("Added assignment %s to gradebook" % assignment_name) except InvalidEntry as e: logger.debug("Error ocurred by adding assignment to gradebook: %s" % e) - return assignment + return assignment \ No newline at end of file diff --git a/src/illumidesk/illumidesk/authenticators/authenticator.py b/src/illumidesk/illumidesk/authenticators/authenticator.py index 6104ba9e..798020ae 100644 --- a/src/illumidesk/illumidesk/authenticators/authenticator.py +++ b/src/illumidesk/illumidesk/authenticators/authenticator.py @@ -60,34 +60,41 @@ async def setup_course_hook_lti11( jupyterhub_api = JupyterHubAPI() # normalize the name and course_id strings in authentication dictionary - username = authentication["name"] + name = authentication["name"] lms_user_id = authentication["auth_state"]["user_id"] + email = authentication["auth_state"]["email"] or lms_user_id user_role = authentication["auth_state"]["roles"].split(",")[0] course_id = lti_utils.normalize_string( authentication["auth_state"]["context_label"] ) - nb_service = NbGraderServiceHelper(course_id, True) + + source = authentication["auth_state"]["source"] or "lti" + source_type = authentication["auth_state"]["source_type"] or "lti11" + nb_service = NbGraderServiceHelper(course_id) # register the user (it doesn't matter if it is a student or instructor) with her/his lms_user_id in nbgrader - nb_service.add_user_to_nbgrader_gradebook(username, lms_user_id) + role = "STUDENT" + if not user_is_a_student(user_role) and user_is_an_instructor(user_role): + role = "INSTRUCTOR" + nb_service.add_user_to_nbgrader_gradebook(email, lms_user_id, source, source_type, role) # TODO: verify the logic to simplify groups creation and membership - if user_is_a_student(user_role): + if role == "STUDENT": try: # assign the user to 'nbgrader-' group in jupyterhub and gradebook - await jupyterhub_api.add_student_to_jupyterhub_group(course_id, username) + await jupyterhub_api.add_student_to_jupyterhub_group(course_id, lms_user_id) except AddJupyterHubUserException as e: logger.error( - "An error when adding student username: %s to course_id: %s with exception %s", - (username, course_id, e), + "An error when adding student user_id: %s to course_id: %s with exception %s", + (lms_user_id, course_id, e), ) - elif user_is_an_instructor(user_role): + elif role == "INSTRUCTOR": try: # assign the user in 'formgrade-' group - await jupyterhub_api.add_instructor_to_jupyterhub_group(course_id, username) + await jupyterhub_api.add_instructor_to_jupyterhub_group(course_id, lms_user_id) except AddJupyterHubUserException as e: logger.error( - "An error when adding instructor username: %s to course_id: %s with exception %s", - (username, course_id, e), + "An error when adding instructor user_id: %s to course_id: %s with exception %s", + (lms_user_id, course_id, e), ) # launch the new grader-notebook as a service @@ -127,31 +134,34 @@ async def setup_course_hook( # normalize the name and course_id strings in authentication dictionary course_id = lti_utils.normalize_string(authentication["auth_state"]["course_id"]) - nb_service = NbGraderServiceHelper(course_id, True) - username = lti_utils.normalize_string(authentication["name"]) + nb_service = NbGraderServiceHelper(course_id) lms_user_id = authentication["auth_state"]["lms_user_id"] + email = authentication["auth_state"]["email"] user_role = authentication["auth_state"]["user_role"] + source = authentication["auth_state"]["source"] + source_type = authentication["auth_state"]["source_type"] # register the user (it doesn't matter if it is a student or instructor) with her/his lms_user_id in nbgrader - nb_service.add_user_to_nbgrader_gradebook(username, lms_user_id) + user = nb_service.add_user_to_nbgrader_gradebook(email, lms_user_id, source, source_type) + authentication["name"] = user["id"] # TODO: verify the logic to simplify groups creation and membership if user_is_a_student(user_role): try: # assign the user to 'nbgrader-' group in jupyterhub and gradebook - await jupyterhub_api.add_student_to_jupyterhub_group(course_id, username) + await jupyterhub_api.add_student_to_jupyterhub_group(course_id, user["id"]) except AddJupyterHubUserException as e: logger.error( - "An error when adding student username: %s to course_id: %s with exception %s", - (username, course_id, e), + "An error when adding student user_id: %s to course_id: %s with exception %s", + (lms_user_id, course_id, e), ) elif user_is_an_instructor(user_role): try: # assign the user in 'formgrade-' group - await jupyterhub_api.add_instructor_to_jupyterhub_group(course_id, username) + await jupyterhub_api.add_instructor_to_jupyterhub_group(course_id, user["id"]) except AddJupyterHubUserException as e: logger.error( - "An error when adding instructor username: %s to course_id: %s with exception %s", - (username, course_id, e), + "An error when adding instructor user_id: %s to course_id: %s with exception %s", + (lms_user_id, course_id, e), ) # launch the new grader-notebook as a service @@ -163,6 +173,57 @@ async def setup_course_hook( return authentication +async def setup_user_hook_auth0( + authenticator: Authenticator, + handler: RequestHandler, + authentication: Dict[str, str], +) -> Dict[str, str]: + """ + Calls the microservice to create a new user in case it does not exist. + The data needed is received from auth_state within authentication object. This + function assumes that the required k/v's in the auth_state dictionary are available, + since the Authenticator(s) validates the data beforehand. + + This function requires `Authenticator.enable_auth_state = True` and is intended + to be used as a post_auth_hook. + + Args: + authenticator: the JupyterHub Authenticator object + handler: the JupyterHub handler object + authentication: the authentication object returned by the + authenticator class + + Returns: + authentication (Required): updated authentication object + """ + + # normalize the name and course_id strings in authentication dictionary + nb_service = NbGraderServiceHelper("default-course") + oauth_user = authentication["auth_state"]["oauth_user"] or {} + + logger.info( + "Auth0 authentication: %s", + (authentication,), + ) + + lms_user_id = oauth_user.get("user_id") or oauth_user.get("username") or oauth_user.get("sub") + email = oauth_user.get("email") or oauth_user.get("sub") + # user_role = authentication["auth_state"]["user_role"] + source = "auth0" + source_type = "generic-oauth" + + # register the user (it doesn't matter if it is a student or instructor) with her/his lms_user_id in nbgrader + user = nb_service.add_user_to_nbgrader_gradebook(email, lms_user_id, source, source_type) + authentication["name"] = user["id"] + # launch the new grader-notebook as a service + try: + _ = await register_new_service(org_name=ORG_NAME, course_id="default-course") + except Exception as e: + logger.error("Unable to launch the shared grader notebook with exception %s", e) + + return authentication + + class LTI13Authenticator(OAuthenticator): """Custom authenticator used with LTI 1.3 requests""" @@ -235,7 +296,9 @@ async def authenticate( # noqa: C901 course_id = lti_utils.normalize_string(course_id) self.log.debug("Normalized course label is %s" % course_id) username = "" + email = "" if "email" in jwt_decoded and jwt_decoded["email"]: + email = jwt_decoded["email"] username = lti_utils.email_to_username(jwt_decoded["email"]) elif "name" in jwt_decoded and jwt_decoded["name"]: username = jwt_decoded["name"] @@ -294,6 +357,7 @@ async def authenticate( # noqa: C901 await process_resource_link_lti_13(self.log, course_id, jwt_decoded) lms_user_id = jwt_decoded["sub"] if "sub" in jwt_decoded else username + lms_id = (jwt_decoded.get("https://purl.imsglobal.org/spec/lti/claim/tool_platform") or {}).get("guid") # ensure the username is normalized self.log.debug("username is %s" % username) @@ -303,14 +367,19 @@ async def authenticate( # noqa: C901 # ensure the user name is normalized username_normalized = lti_utils.normalize_string(username) self.log.debug("Assigned username is: %s" % username_normalized) + self.log.debug("Assigned id is: %s for username %s" % (lms_user_id, username)) return { - "name": username_normalized, + "name": email or lms_user_id, "auth_state": { "course_id": course_id, "user_role": user_role, "lms_user_id": lms_user_id, + "username": username, + "email": email or lms_user_id, "launch_return_url": launch_return_url, + "source": lms_id, + "source_type": "lti13" }, # noqa: E231 } @@ -328,7 +397,7 @@ async def process_resource_link_lti_13( "https://purl.imsglobal.org/spec/lti/claim/resource_link" ] resource_link_title = resource_link["title"] or "" - nbgrader_service = NbGraderServiceHelper(course_id, True) + nbgrader_service = NbGraderServiceHelper(course_id) if resource_link_title: assignment_name = LTIUtils().normalize_string(resource_link_title) logger.debug( diff --git a/src/illumidesk/illumidesk/lti13/handlers.py b/src/illumidesk/illumidesk/lti13/handlers.py index 1fb292fc..63380529 100644 --- a/src/illumidesk/illumidesk/lti13/handlers.py +++ b/src/illumidesk/illumidesk/lti13/handlers.py @@ -199,4 +199,4 @@ async def get(self): files=link_item_files, action_url=auth_state["launch_return_url"], ) - self.finish(html) + self.finish(html) \ No newline at end of file diff --git a/src/illumidesk/requirements.txt b/src/illumidesk/requirements.txt index 3d222b0b..4492fac9 100644 --- a/src/illumidesk/requirements.txt +++ b/src/illumidesk/requirements.txt @@ -28,7 +28,7 @@ ipython-genutils==0.2.0 # via nbformat, notebook, qtconsole, traitlets ipython==7.23.1 # via ipykernel, ipywidgets, jupyter-console ipywidgets==7.6.3 # via jupyter jedi==0.18.0 # via ipython -jinja2==3.0.0 # via jupyterhub, jupyterhub-kubespawner, nbconvert, notebook +jinja2==3.0.3 # via jupyterhub, jupyterhub-kubespawner, nbconvert, notebook josepy==1.4.0 # via illumidesk (setup.py) jsonschema==3.2.0 # via jupyter-telemetry, nbformat, nbgrader jupyter-client==6.1.12 # via ipykernel, jupyter-console, nbgrader, notebook, qtconsole @@ -37,7 +37,7 @@ jupyter-core==4.7.1 # via jupyter-client, nbconvert, nbformat, nbgrader, n jupyter-telemetry==0.1.0 # via jupyterhub jupyter==1.0.0 # via nbgrader jupyterhub-kubespawner==0.14.1 # via illumidesk (setup.py) -git+git://github.com/jupyterhub/ltiauthenticator.git@71d86a9da2562df4bdcc9f374af834a172ac52d5 # via illumidesk (setup.py) +git+https://github.com/jupyterhub/ltiauthenticator.git@71d86a9da2562df4bdcc9f374af834a172ac52d5 # via illumidesk (setup.py) jupyterhub==1.4.1 # via jupyterhub-kubespawner, jupyterhub-ltiauthenticator, oauthenticator jupyterlab-widgets==1.0.0 # via ipywidgets jwcrypto==0.8 # via illumidesk (setup.py) @@ -48,7 +48,7 @@ matplotlib-inline==0.1.2 # via ipython mistune==0.8.4 # via nbconvert nbconvert==5.6.1 # via jupyter, nbgrader, notebook nbformat==5.1.3 # via ipywidgets, nbconvert, nbgrader, notebook -nbgrader==0.6.2 # via illumidesk (setup.py) +git+ssh://git@github.com/IllumiDesk/illumidesk-next.git@feature/refactor-modals-lti#subdirectory=packages/nbgrader # via illumidesk (setup.py) notebook==6.4.1 # via jupyter, nbgrader, widgetsnbextension oauthenticator==14.2.0 # via illumidesk (setup.py) oauthlib==3.1.0 # via illumidesk (setup.py), jupyterhub, jupyterhub-ltiauthenticator, requests-oauthlib @@ -102,6 +102,7 @@ wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 # via bleach websocket-client==0.59.0 # via kubernetes widgetsnbextension==3.5.1 # via ipywidgets +secretsmanager-illumidesk==0.0.3 # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/illumidesk/setup.py b/src/illumidesk/setup.py index b0cd6319..58aececa 100644 --- a/src/illumidesk/setup.py +++ b/src/illumidesk/setup.py @@ -38,9 +38,10 @@ install_requires=[ "josepy==1.4.0", "jupyterhub-kubespawner==0.14.1", - "jupyterhub-ltiauthenticator@git+git://github.com/jupyterhub/ltiauthenticator.git@71d86a9da2562df4bdcc9f374af834a172ac52d5", + "jupyterhub-ltiauthenticator==1.3.0", "jwcrypto==0.8", - "nbgrader==0.6.2", + "secretsmanager-illumidesk==0.0.3", + "nbgrader @ git+ssh://git@github.com/IllumiDesk/illumidesk-next.git@feature/refactor-modals-lti#subdirectory=packages/nbgrader&egg=nbgrader" "oauthlib==3.1", "oauthenticator>=0.13.0", "pem==20.1.0", diff --git a/src/illumidesk/tests/illumidesk/apis/test_nbgrader_service_helper.py b/src/illumidesk/tests/illumidesk/apis/test_nbgrader_service_helper.py index 16fac5f4..1f82c6e5 100644 --- a/src/illumidesk/tests/illumidesk/apis/test_nbgrader_service_helper.py +++ b/src/illumidesk/tests/illumidesk/apis/test_nbgrader_service_helper.py @@ -37,10 +37,10 @@ def test_add_user_to_nbgrader_gradebook_raises_error_when_empty( Does add_user_to_nbgrader_gradebook method accept an empty username, or lms user id? """ with pytest.raises(ValueError): - self.sut.add_user_to_nbgrader_gradebook(username="", lms_user_id="abc123") + self.sut.add_user_to_nbgrader_gradebook(email="", external_user_id="abc123", source="test", source_type="test") with pytest.raises(ValueError): - self.sut.add_user_to_nbgrader_gradebook(username="user1", lms_user_id="") + self.sut.add_user_to_nbgrader_gradebook(email="user1", external_user_id="", source="test", source_type="test") class TestNbGraderServiceHelper: @@ -56,9 +56,9 @@ def test_nbgrader_format_db_url_method_uses_env_vars_to_get_db_url( monkeypatch.setattr( "illumidesk.apis.nbgrader_service.nbgrader_db_user", "test_user" ) - monkeypatch.setattr("illumidesk.apis.nbgrader_service.org_name", "org-dummy") + monkeypatch.setattr("illumidesk.apis.nbgrader_service.nbgrader_db_name", "dummy_database") assert ( - nbgrader_format_db_url("Course 1") - == "postgresql://test_user:test_pwd@test_host:5432/org-dummy_course1" + nbgrader_format_db_url() + == "postgresql://test_user:test_pwd@test_host:5432/dummy_database" ) diff --git a/src/illumidesk/tests/illumidesk/conftest.py b/src/illumidesk/tests/illumidesk/conftest.py index 3774b4a0..790a23b7 100644 --- a/src/illumidesk/tests/illumidesk/conftest.py +++ b/src/illumidesk/tests/illumidesk/conftest.py @@ -27,7 +27,7 @@ @pytest.fixture(scope="module") def auth_state_dict(): authenticator_auth_state = { - "name": "student1", + "name": "185d6c59731a553009ca9b59ca3a885100000", "auth_state": { "course_id": "intro101", "lms_user_id": "185d6c59731a553009ca9b59ca3a885100000", @@ -63,7 +63,6 @@ def mock_nbhelper(): "illumidesk.apis.nbgrader_service.NbGraderServiceHelper", # __init__=lambda x, y: None, update_course=Mock(return_value=None), - create_database_if_not_exists=Mock(), add_user_to_nbgrader_gradebook=Mock(return_value=None), register_assignment=Mock(return_value=None), get_course=Mock(