developers: create a a dev environment and fix the db-proxy with it

This commit moves some "default" values that are aimed only at production
into the secrets.yaml file (which overrides values.yaml on install). That
allows values.yaml to contain dummy values and defaults that allow installing
disposable dev environments from helm without modification.

This adds a NOTES.txt file, which is templated out and is displayed on
"helm install" so that specific commands and values can be shown to the
user to complete their dev environment.

Since Bitnami has the only helm repository with a public and functioning
mediawiki, we are using that for mediawiki and mariadb for now. They are
widely used charts that are well-regarded and professionally maintained.

Bug: T260389

.gitignore

@@ -5,3 +5,4 @@ __pycache__/
 .mypy_cache/
 charts/
 requirements.lock
+dev-values.yaml

README.md

@@ -13,6 +13,28 @@ If you have push access to the project, we ask that new changes be reviewed by o
 project member by using either a feature branch on the  https://github.com/toolforge/paws repo
 to trigger a pull request or using a fork to set up a pull request.
 
+### Settings up a development environment
+
+It is possible to run a fully-functioning PAWS system inside [minikube](https://minikube.sigs.k8s.io/docs/)! You don't need
+access to the secrets.yaml file to do it either, since the defaults mostly support it.
+
+You will need to install minikube and [helm](https://helm.sh) on your system. When you are confident those are working,
+you need to create a new namespace for paws (`kubectl create namespace paws-dev` for instance) and then install into it
+with helm (from the top level of this repo):
+`helm -n paws-dev install dev paws/`
+
+The rest of the setup instructions will display on screen as long as the install is successful.
+Please refer to the helm documentation from there.
+
+NOTE: By default the mariadb chart keeps a PersistentVolumeClaim around for its storage even after
+uninstall. If you intend on rebuilding your dev environment later, you will need to use all the same
+values for DB and DB passwords if you don't delete that claim and volume (and the data from your
+last wiki will be in there--which means you keep your oauth grant!). The PVC for mediawiki gets cleaned up on uninstall.
+
+If minikube is acting weird, it might be worth it to upgrade minikube or even to
+ increase the default memory:
+`minikube config set memory 4096`
+
 ## Useful libraries
 ### Accessing Database Replicas With Pandas and Sqlalchemy
 

images/db-proxy/auth.lua

@@ -23,7 +23,7 @@ local proto = require("mysql.proto")
 package.cpath = package.cpath .. ';/usr/local/lib/lua/5.1/?.so'
 local crypto = require('crypto')
 local cjson = require('cjson')
-local socket = require('socket')
+local socket = require('socket.core')
 
 local HMAC_KEY = os.getenv('HMAC_KEY')
 local MYSQL_USERNAME = os.getenv('MYSQL_USERNAME')
@@ -37,16 +37,16 @@ function connect_server()
 
 	if (database == 'enwiki_p') then
 		proxy.connection.backend_ndx = 1
-	else if (database == 'wikidatawiki_p') then
+	elseif (database == 'wikidatawiki_p') then
 		proxy.connection.backend_ndx = 8
-	else if (database == 'commonswiki_p' or database == 'testcommonswiki_p') then
+	elseif (database == 'commonswiki_p' or database == 'testcommonswiki_p') then
 		proxy.connection.backend_ndx = 4
-	else if (database == 'meta_p') then
+	elseif (database == 'meta_p') then
 		proxy.connection.backend_ndx = 7
 	else
 		-- If this isn't a well-known db, ask DNS!
 		local source_db = string.gsub(database, "_p", "")
-		local db_host = source_db .. '.' .. MYSQLDOMAIN
+		local db_host = source_db .. '.' .. MYSQL_DOMAIN
 		local db_address, address_info = dns.toip(db_host)
 		-- backend_address should be the canonical name (eg. s4.analytics...)
 		local backend_address = address_info.name

images/jobber/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.12
+
+RUN apk --no-cache add curl jq
+
+COPY install-oauth.sh /opt/install-oauth.sh
+COPY hack-localsettings.sh /opt/hack-localsettings.sh
+
+RUN chmod a+x /opt/install-oauth.sh
+RUN chmod a+x /opt/hack-localsettings.sh
+
+WORKDIR /opt/mediawiki

images/jobber/hack-localsettings.sh

@@ -0,0 +1,17 @@
+#!/bin/ash
+
+if ! test -f /opt/mediawiki/LocalSettings.php; then
+    echo "The settings file LocalSettings.php is missing!"
+    exit 1
+fi
+
+/bin/cat <<"EOF" >> /opt/mediawiki/LocalSettings.php
+wfLoadExtension( 'OAuth' );
+$wgMWOAuthSecureTokenTransfer = false;
+$wgEmailAuthentication = false;
+$wgOAuthSecretKey = '0469807d667f4d6cdbf3ae772ea874d95518fbe41c59f73eb59169f7ed02b7d3';
+$wgGroupPermissions['user']['mwoauthmanageconsumer'] = true;
+$wgGroupPermissions['user']['mwoauthproposeconsumer'] = true;
+$wgGroupPermissions['user']['mwoauthupdateownconsumer'] = true;
+$wgOAuthGroupsToNotify = [ 'sysop' ];
+EOF

images/jobber/install-oauth.sh

@@ -0,0 +1,11 @@
+#!/bin/ash
+
+extension_url=$(curl 'https://www.mediawiki.org/w/api.php?action=query&list=extdistbranches&edbexts=OAuth&formatversion=2&format=json' | jq -r .query.extdistbranches.extensions.OAuth.REL1_34)
+/usr/bin/curl -o OAuth.tar.gz "$extension_url"
+
+# Wait until the mediawiki pod creates the extensions dir in the PVC
+while ! test -d "/opt/mediawiki/extensions"; do
+    sleep 1
+done
+tar -xzf OAuth.tar.gz -C /opt/mediawiki/extensions
+chown -R 1001:0 /opt/mediawiki/extensions/OAuth

paws/requirements.yaml

@@ -2,3 +2,9 @@ dependencies:
   - name: jupyterhub
     version: "0.9.0"
     repository: "https://jupyterhub.github.io/helm-chart"
+  - name: mediawiki
+    version: "10.x.x"
+    repository: "https://charts.bitnami.com/bitnami"
+    condition: mediawiki.enabled
+    tags:
+      - testenv

paws/templates/NOTES.txt

@@ -0,0 +1,49 @@
+Thank you for installing {{ .Chart.Name }}!
+{{- if or .Values.mediawiki.enabled }}
+
+An essential part of setting up your dev environment is adding a hosts file entry.
+Get the IP of minikube with:
+minikube ip
+Add these lines to your hosts file:
+<ip address> {{ (index .Values.mediawiki.ingress.hosts 0).name }}
+<ip address> {{ index .Values.jupyterhub.ingress.hosts 0 }}
+
+If you have never run the DB updates for the OAuth extension for your local
+Mediawiki instance. If this is true, you might want to delete the mediawiki pod to restart it
+with the new config (just in case) with:
+kubectl -n {{ .Release.Namespace }} delete $(kubectl get pods -n {{ .Release.Namespace }} -l app=mediawiki -o name)
+
+After giving that a chance to start back up, go to http://{{ (index .Values.mediawiki.ingress.hosts 0).name }}/mw_config/
+
+From here follow the instructions at https://www.mediawiki.org/wiki/Manual:Upgrading#Web_browser
+to run the update.php script via your browser. This will create the necessary tables.
+
+You will need the value of $wgUpgradeKey from LocalSettings.php to run it. To get that,
+you can try running the following:
+kubectl -n {{ .Release.Namespace }} exec $(kubectl get pods -n {{ .Release.Namespace }} -l app=mediawiki -o name) -- cat /opt/bitnami/mediawiki/LocalSettings.php | grep wgUpgradeKey
+
+When that is done, you'll need to make an OAuth consumer in your local wiki.
+Your wiki login will be 
+username: {{ .Values.mediawiki.mediawikiUser }}
+password: {{ .Values.mediawiki.mediawikiPassword }}
+
+If you have been following these directions, go to http://{{ (index .Values.mediawiki.ingress.hosts 0).name }}/wiki/Special:OAuthConsumerRegistration to create one.
+You must write down the output from that request setup or you won't get a second look at your secret easily.
+You'll need to approve it at http://{{ (index .Values.mediawiki.ingress.hosts 0).name }}/wiki/Special:OAuthManageConsumers after that.
+
+Now create a file called dev-values.yaml and add to it:
+jupyterhub:
+  hub:
+    extraEnv:
+      MW_INDEX_URL: http://{{ (index .Values.mediawiki.ingress.hosts 0).name }}/index.php
+  auth:
+    mediawiki:
+      clientId: <the client ID from your OAuth Consumer request>
+      clientSecret: <the client secret from your OAuth Consumer request>
+      indexUrl: http://{{ (index .Values.mediawiki.ingress.hosts 0).name }}/index.php
+
+Now run:
+helm -n {{ .Release.Namespace }} upgrade {{ .Release.Name }} paws/ -f dev-values.yaml
+Then you should be able to go to {{ index .Values.jupyterhub.ingress.hosts 0 }} and log in!
+Happy hacking.
+{{- end }}

paws/templates/db-proxy.yaml

@@ -1,4 +1,3 @@
-{{ if .Values.mysqlProxy }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -17,6 +16,24 @@ spec:
       containers:
       - name: db-proxy
         image: {{ tpl .Values.dbProxy.image.template . | quote }}
+        imagePullPolicy: IfNotPresent
+        command:
+          - mysql-proxy
+          - --plugins=proxy
+          - --proxy-lua-script=/srv/auth.lua
+          {{- if .Values.mediawiki.enabled }}
+          - {{ printf "--proxy-backend-addresses=%s-mariadb.%s:3306" .Release.Name .Release.Namespace | quote }}
+          {{- else }}
+          {{- with .Values.mysql }}
+          {{- $sections := untilStep 1 9 1 | toStrings }}
+          {{- $domain := .domain }}
+          {{- range $sections }}
+          - {{ printf "--proxy-backend-addresses=s%s.%s:3306" . $domain | quote }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          - --proxy-address=0.0.0.0:3306
+          - --proxy-skip-profiling
         env:
           {{- with .Values.mysql }}
           {{- $sections := untilStep 1 9 1 | toStrings }}
@@ -51,14 +68,3 @@ spec:
     targetPort: 3306
   selector:
     name: db-proxy
-{{ else }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: mysql
-  labels:
-    app: wikireplicas
-spec:
-  type: ExternalName
-  externalName: {{ .Values.mysql.host | quote }}
-{{ end }}

paws/templates/mediawiki-hacks.yaml

@@ -0,0 +1,147 @@
+{{ if .Values.mediawiki.enabled }}
+# We assume mediawiki is only enabled in a local dev environment, so...
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: scary-host-vol-builder
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: {{ printf "host-vols-%s" .Release.Name | quote }}
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: creating-host-vols
+        image: {{ tpl .Values.mediawikiHacks.image.template . | quote }}
+        command: 
+          - mkdir
+          - -p
+          - /mnt/mnt/nfs/dumps-labstore1006.wikimedia.org
+          - /mnt/mnt/nfs/dumps-labstore1007.wikimedia.org
+          - /mnt/public/dumps
+          - /mnt/data/project/paws/userhomes
+        volumeMounts:
+          - mountPath: /mnt
+            name: security-disaster-only-for-minikube
+      volumes:
+        - name: security-disaster-only-for-minikube
+          hostPath:
+            path: /
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: scary-host-vol-perm-fix
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: {{ printf "permission-fix-%s" .Release.Name | quote }}
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: fixingperms-host-vols
+        image: {{ tpl .Values.mediawikiHacks.image.template . | quote }}
+        command: 
+          - chown
+          - -R
+          - 52771:52771
+          - /mnt/data/project/paws/userhomes
+        volumeMounts:
+          - mountPath: /mnt
+            name: security-disaster-only-for-minikube
+      volumes:
+        - name: security-disaster-only-for-minikube
+          hostPath:
+            path: /
+---
+# minikube provisions a hostpath where extensions and config can be placed
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mediawiki-hacks
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: {{ printf "mediawiki-hacks-%s" .Release.Name | quote }}
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: mediawiki-config-hacks
+        image: {{ tpl .Values.mediawikiHacks.image.template . | quote }}
+        command: ["/opt/hack-localsettings.sh"]
+        volumeMounts:
+          - mountPath: /opt/mediawiki
+            name: mediawiki-data
+            subPath: mediawiki
+      volumes:
+        - name: mediawiki-data
+          persistentVolumeClaim:
+            claimName: {{ if .Values.mediawiki.persistence.existingClaim }}{{ .Values.mediawiki.persistence.existingClaim }}{{- else }}{{ template "mediawiki.fullname" . }}-mediawiki-mediawiki{{- end }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mediawiki-oauth-install
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      name: {{ printf "mediawiki-extension-%s" .Release.Name | quote }}
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: mediawiki-extension-install
+        image: {{ tpl .Values.mediawikiHacks.image.template . | quote }}
+        command: ["/opt/install-oauth.sh"]
+        volumeMounts:
+          - mountPath: /opt/mediawiki
+            name: mediawiki-data
+            subPath: mediawiki
+      volumes:
+        - name: mediawiki-data
+          persistentVolumeClaim:
+            claimName: {{ if .Values.mediawiki.persistence.existingClaim }}{{ .Values.mediawiki.persistence.existingClaim }}{{- else }}{{ template "mediawiki.fullname" . }}-mediawiki-mediawiki{{- end }}
+{{ end }}