From 7eb78504578ba99d6cc8fb122743fad2c4646016 Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Tue, 28 Sep 2021 12:29:00 -0400 Subject: [PATCH 1/8] feat: add analyticsapi service --- Makefile | 3 +++ check.sh | 6 ++++++ docker-compose.yml | 20 ++++++++++++++++++++ options.mk | 2 +- provision-analyticsapi.sh | 25 +++++++++++++++++++++++++ provision-insights.sh | 2 +- provision.sh | 1 + provision.sql | 12 +++++++++++- 8 files changed, 68 insertions(+), 3 deletions(-) create mode 100755 provision-analyticsapi.sh diff --git a/Makefile b/Makefile index 2f1eac7bf0..74efca352f 100644 --- a/Makefile +++ b/Makefile @@ -442,6 +442,9 @@ dev.shell.studio_watcher: dev.shell.xqueue_consumer: docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open +dev.shell.analyticsapi: + docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) /edx/app/analytics_api/devstack.sh open + dev.shell.insights: docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /edx/app/insights/devstack.sh open' diff --git a/check.sh b/check.sh index 6087ae989d..6fb20f3833 100755 --- a/check.sh +++ b/check.sh @@ -115,6 +115,12 @@ if should_check insights; then "curl --fail -L http://localhost:18110/health/" fi +if should_check analyticsapi; then + echo "Running Analytics Data API Devstack tests: " + run_check analyticsapi_heartbeat analyticsapi \ + "curl --fail -L http://localhost:18100/health/" +fi + echo "Successful checks:${succeeded:- NONE}" echo "Failed checks:${failed:- NONE}" if [[ "$succeeded" ]]; then diff --git a/docker-compose.yml b/docker-compose.yml index f03a67427c..97015bc0f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -356,6 +356,7 @@ services: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights" hostname: insights.devstack.edx depends_on: + - analyticsapi - mysql57 - lms - memcached @@ -380,6 +381,25 @@ services: volumes: - /edx/var/insights/ + analyticsapi: + image: edxops/analytics_api:${OPENEDX_RELEASE:-latest} + container_name: edx.devstack.analyticsapi + hostname: analyticsapi + depends_on: + - mysql57 + - elasticsearch710 + command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' + stdin_open: true + tty: true + environment: + DB_HOST: edx.devstack.mysql57 + DB_PORT: 3306 + ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710 + volumes: + - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api + ports: + - "19001:19001" + registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar" diff --git a/options.mk b/options.mk index bb0a320d83..2e11b83012 100644 --- a/options.mk +++ b/options.mk @@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-gradebook+front # Separated by plus signs. # Separated by plus signs. Listed in alphabetical order for clarity. EDX_SERVICES ?= \ -credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer +analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer # Services with database migrations. # Should be a subset of $(EDX_SERVICES). diff --git a/provision-analyticsapi.sh b/provision-analyticsapi.sh new file mode 100755 index 0000000000..deb0b8bc2d --- /dev/null +++ b/provision-analyticsapi.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +. scripts/colors.sh +set -x + +name=analyticsapi +port=19001 + +docker-compose up -d ${name} + +echo -e "${GREEN}Installing requirements for ${name}...${NC}" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name} + +echo -e "${GREEN}Running migrations for ${name}...${NC}" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name} + +echo -e "${GREEN}Creating default user and authentication token for ${name}...${NC}" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name} + +echo -e "${GREEN}Loading test data for ${name}...${NC}" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} + +echo -e "${GREEN}Populating elasticsearch for ${name}...${NC}" +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make create-indices' -- ${name} diff --git a/provision-insights.sh b/provision-insights.sh index 7fade46291..88992075dd 100755 --- a/provision-insights.sh +++ b/provision-insights.sh @@ -5,7 +5,7 @@ set -eu -o pipefail set -x name=insights -port=8011 +port=18011 docker-compose up -d insights diff --git a/provision.sh b/provision.sh index a208f9f09f..849d5a066f 100755 --- a/provision.sh +++ b/provision.sh @@ -49,6 +49,7 @@ registrar \ xqueue \ coursegraph \ insights \ +analyticsapi \ " # What should we provision? diff --git a/provision.sql b/provision.sql index 9434c27ef9..05cb9ec4d4 100644 --- a/provision.sql +++ b/provision.sql @@ -22,6 +22,16 @@ GRANT ALL ON edxapp.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; GRANT ALL ON edxapp_csmh.* TO 'edxapp001'@'%'; CREATE DATABASE IF NOT EXISTS `dashboard`; -GRANT ALL PRIVILEGES ON `dashboard`.* TO 'rozencrantz'@'%' IDENTIFIED BY 'secret'; +GRANT ALL ON `dashboard`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON `dashboard`.* TO 'rozencrantz'@'%' IDENTIFIED BY 'secret'; + +CREATE DATABASE IF NOT EXISTS `analytics-api`; +GRANT ALL ON `analytics-api`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON `analytics-api`.* TO 'api001'@'%' IDENTIFIED BY 'password'; + +CREATE DATABASE IF NOT EXISTS `reports`; +GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL ON `reports`.* TO 'reports001'@'%' IDENTIFIED BY 'password'; + FLUSH PRIVILEGES; From 571697f16cc396682611e4afa050f5274202a637 Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Wed, 20 Oct 2021 15:01:52 -0400 Subject: [PATCH 2/8] build: analytics db vars --- docker-compose.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 97015bc0f1..0e9d3d25ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,8 +173,8 @@ services: default: aliases: - edx.devstack.mysql57 - # ports: - # - "3506:3306" + ports: + - "3506:3306" volumes: - mysql57_data:/var/lib/mysql @@ -367,8 +367,8 @@ services: DB_HOST: edx.devstack.mysql57 DB_NAME: dashboard DB_PORT: 3306 - DB_USER: rozencrantz - DB_PASSWORD: secret + DB_USER: analytics001 + DB_PASSWORD: password LMS_HOST: http://localhost:18000 DJANGO_SETTINGS_MODULE: analytics_dashboard.settings.devstack image: edxops/insights:${OPENEDX_RELEASE:-latest} @@ -394,6 +394,8 @@ services: environment: DB_HOST: edx.devstack.mysql57 DB_PORT: 3306 + DB_USER: analytics001 + DB_PASSWORD: password ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710 volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api From f28f22254e22c1c9add15d0dba657ecdd9f27b9a Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Wed, 20 Oct 2021 16:10:42 -0400 Subject: [PATCH 3/8] build: fix analyticsdata service --- docker-compose-host.yml | 3 +++ provision-analyticsapi.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose-host.yml b/docker-compose-host.yml index 3e015ab755..dec26dc302 100644 --- a/docker-compose-host.yml +++ b/docker-compose-host.yml @@ -53,6 +53,9 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights - insights_node_modules:/edx/app/insights/insights/node_modules + analyticsapi: + volumes: + - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api # Note that frontends mount `src` to /edx/app/src instead of /edx/src. # See ADR #5 for rationale. diff --git a/provision-analyticsapi.sh b/provision-analyticsapi.sh index deb0b8bc2d..7cd6f0105b 100755 --- a/provision-analyticsapi.sh +++ b/provision-analyticsapi.sh @@ -22,4 +22,4 @@ echo -e "${GREEN}Loading test data for ${name}...${NC}" docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} echo -e "${GREEN}Populating elasticsearch for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make create-indices' -- ${name} +docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make create_indices' -- ${name} From de047417a77efb1cba708af2c17071681968cca4 Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Thu, 21 Oct 2021 09:31:13 -0400 Subject: [PATCH 4/8] docs: add analyticsapi to readme --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 9a8c18795f..d07778b61e 100644 --- a/README.rst +++ b/README.rst @@ -322,6 +322,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro +------------------------------------+-------------------------------------+----------------+--------------+ | `insights` | http://localhost:18110 | Python/Django | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ +| `analyticsapi` | http://localhost:19001 | Python/Django | Extra | ++------------------------------------+-------------------------------------+----------------+--------------+ | `frontend-app-ora-grading` | http://localhost:1993 | MFE (React.js) | Extra | +------------------------------------+-------------------------------------+----------------+--------------+ From 61eac5b479d410f5c354c9867db125dab6148162 Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Thu, 21 Oct 2021 10:07:34 -0400 Subject: [PATCH 5/8] build: remove old analytics db users --- provision.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/provision.sql b/provision.sql index 05cb9ec4d4..d2676c6fe1 100644 --- a/provision.sql +++ b/provision.sql @@ -23,15 +23,12 @@ GRANT ALL ON edxapp_csmh.* TO 'edxapp001'@'%'; CREATE DATABASE IF NOT EXISTS `dashboard`; GRANT ALL ON `dashboard`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; -GRANT ALL ON `dashboard`.* TO 'rozencrantz'@'%' IDENTIFIED BY 'secret'; CREATE DATABASE IF NOT EXISTS `analytics-api`; GRANT ALL ON `analytics-api`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; -GRANT ALL ON `analytics-api`.* TO 'api001'@'%' IDENTIFIED BY 'password'; CREATE DATABASE IF NOT EXISTS `reports`; GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; -GRANT ALL ON `reports`.* TO 'reports001'@'%' IDENTIFIED BY 'password'; FLUSH PRIVILEGES; From aad04c0dde75a8fd620a23b78ececd662e602590 Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Thu, 21 Oct 2021 10:56:30 -0400 Subject: [PATCH 6/8] fix: comment mysql57 ports --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e9d3d25ed..4fd89b7565 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,8 +173,8 @@ services: default: aliases: - edx.devstack.mysql57 - ports: - - "3506:3306" + # ports: + # - "3506:3306" volumes: - mysql57_data:/var/lib/mysql From 17601e3121e87e28c5ae9d4cc6f492bbfdf8dbfd Mon Sep 17 00:00:00 2001 From: Zach Hancock Date: Thu, 21 Oct 2021 12:09:01 -0400 Subject: [PATCH 7/8] docs: missed readme items --- README.rst | 2 ++ docker-compose.yml | 4 ++-- repo.sh | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d07778b61e..c41b03a65b 100644 --- a/README.rst +++ b/README.rst @@ -353,6 +353,8 @@ Some common service combinations include: .. _xqueue: https://github.com/edx/xqueue .. _coursegraph: https://github.com/edx/edx-platform/tree/master/openedx/core/djangoapps/coursegraph .. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading +.. _insights: https://github.com/edx/edx-analytics-dashboard +.. _analyticsapi: https://github.com/edx/edx-analytics-data-api Known Issues diff --git a/docker-compose.yml b/docker-compose.yml index 4fd89b7565..829a15735f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -397,10 +397,10 @@ services: DB_USER: analytics001 DB_PASSWORD: password ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710 - volumes: - - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api ports: - "19001:19001" + volumes: + - /edx/var/analyticsapi registrar: command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done' diff --git a/repo.sh b/repo.sh index 09ecfc8011..c525f8432c 100755 --- a/repo.sh +++ b/repo.sh @@ -33,6 +33,8 @@ repos=( "https://github.com/edx/frontend-app-gradebook.git" "https://github.com/edx/frontend-app-payment.git" "https://github.com/edx/frontend-app-publisher.git" + "https://github.com/edx/edx-analytics-dashboard.git" + "https://github.com/edx/edx-analytics-data-api.git" ) non_release_repos=( @@ -58,6 +60,8 @@ ssh_repos=( "git@github.com:edx/frontend-app-gradebook.git" "git@github.com:edx/frontend-app-payment.git" "git@github.com:edx/frontend-app-publisher.git" + "git@github.com:edx/edx-analytics-dashboard.git" + "git@github.com:edx/edx-analytics-data-api.git" ) non_release_ssh_repos=( From 1b00fecdd954f050d28b5dc33305683ce2493067 Mon Sep 17 00:00:00 2001 From: Zachary Hancock Date: Mon, 25 Oct 2021 10:23:40 -0400 Subject: [PATCH 8/8] fix: analytics check.sh port Co-authored-by: Kyle McCormick --- check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check.sh b/check.sh index 6fb20f3833..ab32687ee2 100755 --- a/check.sh +++ b/check.sh @@ -118,7 +118,7 @@ fi if should_check analyticsapi; then echo "Running Analytics Data API Devstack tests: " run_check analyticsapi_heartbeat analyticsapi \ - "curl --fail -L http://localhost:18100/health/" + "curl --fail -L http://localhost:19001/health/" fi echo "Successful checks:${succeeded:- NONE}"