wmde · m90 · Mar 28, 2023 · Mar 28, 2023
diff --git a/k8s/helmfile/env/local/api.values.yaml.gotmpl b/k8s/helmfile/env/local/api.values.yaml.gotmpl
@@ -1,10 +1,15 @@
 image:
-  tag: "8x.10.1"
+  tag: "8x.11.0"
 
 ingress:
   tls: null
 
+platform:
+  backendMwHost: mediawiki-139-app-backend.default.svc.cluster.local
+
 wbstack:
+  wikiDbProvisionVersion: mw1.39-wbs1
+  wikiDbUseVersion: mw1.39-wbs1
   maxWikisPerUser: null
   elasticSearch:
     enabledByDefault: true

diff --git a/k8s/helmfile/env/local/mediawiki-139.values.yaml.gotmpl b/k8s/helmfile/env/local/mediawiki-139.values.yaml.gotmpl
@@ -0,0 +1,23 @@
+image:
+  tag: "1.39-7.4-20230328-0"
+
+mw:
+  settings:
+    # Enable this to increase verbosity of logging in mw pods
+    # This is very useful when debugging locally but may interfere with jobs
+    logToStdErr: false
+    allowedProxyCidr: null
+  mail:
+    domain: examplemaildomain.localhost
+  smtp:
+    enabled: true
+    host: mailhog
+    port: 1025
+    auth: false
+    smtpUserSecretName: null
+    smtpPasswordSecretName: null
+  recaptcha:
+    sitekeySecretName: recaptcha-v2-dev-secrets
+    sitekeySecretKey: site_key
+    secretkeySecretName: recaptcha-v2-dev-secrets
+    secretkeySecretKey: secret_key
diff --git a/k8s/helmfile/env/local/platform-nginx.nginx.conf b/k8s/helmfile/env/local/platform-nginx.nginx.conf
@@ -0,0 +1,152 @@
+# When listing a BUNCH of stuff in a map, this needed bumping..
+map_hash_bucket_size 128;
+
+# For current deployement this is "app", in the future this can be the version..
+#SELECT CONCAT("\"", domain, "\""), "\"app\";"
+#FROM `wikis`
+#WHERE deleted_at IS NULL
+#LIMIT 900
+map $host $mwversion {
+        default "139-app";
+}
+# Figure out which group of backends we might want to send the request to based on uri
+# TODO add Special:EntityData???
+map $request_uri $mwgroup {
+        ~^/()(w/load.php.*) "web";
+        ~^/()(w/api.php.*) "api";
+        ~^/()(w/rest.php.*) "api";
+        default "web";
+}
+
+server {
+        listen 8080;
+        server_name _; # This is just an invalid value which will never trigger on a real hostname.
+
+        # Resolver is needed when using variables in proxy_pass directives...
+        # https://serverfault.com/a/937172
+        # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/795#issuecomment-451257479
+        resolver kube-dns.kube-system.svc.cluster.local valid=10s;
+
+        # IP range matches current kubernetes pod IPs
+        set_real_ip_from  10.0.0.0/14;
+        real_ip_header X-Forwarded-For;
+        proxy_set_header X-Forwarded-For "$proxy_add_x_forwarded_for";
+
+        client_max_body_size                    1m;
+
+        port_in_redirect off;
+
+        proxy_set_header Host                   $http_host;
+
+        # mitigate HTTPoxy Vulnerability
+        # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
+        proxy_set_header Proxy                  "";
+
+        proxy_redirect                          off;
+        proxy_request_buffering                 on;
+        proxy_http_version                      1.1;
+
+        proxy_cookie_domain                     off;
+        proxy_cookie_path                       off;
+
+        # In case of errors try the next upstream server before returning an error
+        proxy_next_upstream                     error timeout;
+        proxy_next_upstream_timeout             0;
+        proxy_next_upstream_tries               3;
+
+        # Custom headers to proxied server
+        proxy_connect_timeout                   5s;
+        proxy_send_timeout                      60s;
+        proxy_read_timeout                      60s;
+
+        proxy_buffering                         off;
+        proxy_buffer_size                       4k;
+        proxy_buffers                           4 4k;
+
+        proxy_max_temp_file_size                1024m;
+
+        # Allowoverriding the group decision using the magic header
+        if ($http_x_wbstack_alpha) {
+                set $mwgroup "alpha";
+        }
+
+        ############# Locations #############
+
+        location = /kube-probe {
+                add_header Content-Type text/plain;
+                return 200 'gangnam style!';
+        }
+
+        location ~* "^/(query/)(sparql.*)" {
+                ########################################
+                # --- START Blazegraph specific stuff ---
+                ########################################
+
+                # Make sure the blazegraph service is readonly
+                # TODO maybe the headers and the only GET AND POST should mvoe to the gateway?
+                proxy_set_header X-BIGDATA-MAX-QUERY-MILLIS 60000;
+                proxy_set_header X-BIGDATA-READ-ONLY "yes";
+                limit_except GET OPTIONS POST {
+                    deny all;
+                }
+
+                add_header Access-Control-Allow-Origin * always;
+
+                # Cache on clients for 5 mins
+                add_header Cache-Control "public, max-age=300";
+                add_header Vary Accept;
+
+                # Proxy result buffering (not even sure if we want this)
+                client_body_buffer_size 1m;
+                proxy_buffering on;
+                proxy_buffer_size 16k;
+                proxy_buffers 256 16k;
+                proxy_busy_buffers_size 256k;
+                proxy_temp_file_write_size 16k;
+                proxy_max_temp_file_size 10m;
+                proxy_read_timeout 300;
+
+                ########################################
+                # --- END Blazegraph specific stuff ---
+                ########################################
+
+                rewrite "(?i)/(query/)(sparql.*)" /$2 break;
+                proxy_pass http://queryservice-gateway.default.svc.cluster.local:80;
+
+        }
+
+        location ~* "^/(query)(.*)" {
+                rewrite ^(/query)$ $1/ permanent;
+                rewrite "(?i)/(query)(.*)" /$2 break;
+                proxy_pass http://queryservice-ui:80;
+
+        }
+
+        location ~* "^/(tools/widar)(.*)" {
+                rewrite ^(/tools/widar)$ $1/ permanent;
+                rewrite "(?i)/(tools/widar)(.*)" /$2 break;
+                proxy_pass http://tool-widar:80;
+
+        }
+
+        location ~* "^/(tools/cradle)(.*)" {
+                rewrite ^(/tools/cradle)$ $1/ permanent;
+                rewrite "(?i)/(tools/cradle)(.*)" /$2 break;
+                proxy_pass http://tool-cradle:80;
+
+        }
+
+        location ~* "^/(tools/quickstatements)(.*)" {
+                rewrite ^(/tools/quickstatements)$ $1/ permanent;
+                rewrite "(?i)/(tools/quickstatements)(.*)" /$2 break;
+                proxy_pass http://tool-quickstatements:80;
+        }
+
+        location ~* "^/()(.*)" {
+                add_header X-WBSTACK-MW-BACKEND mediawiki-$mwversion-$mwgroup;
+                proxy_pass http://mediawiki-$mwversion-$mwgroup.default.svc.cluster.local:80;
+                rewrite "(?i)/()(.*)" /$2 break;
+
+        }
+
+}
diff --git a/k8s/helmfile/env/local/platform-nginx.values.yaml.gotmpl b/k8s/helmfile/env/local/platform-nginx.values.yaml.gotmpl
@@ -1 +1,4 @@
 replicaCount: 1
+
+serverBlock: |-
+{{ readFile "platform-nginx.nginx.conf" | indent 2 }}
diff --git a/k8s/helmfile/env/local/tool-quickstatements.values.yaml.gotmpl b/k8s/helmfile/env/local/tool-quickstatements.values.yaml.gotmpl
@@ -1 +1,2 @@
-# the staging service matches production, so this file is empty
+platform:
+  mediawikiBackendHost: mediawiki-139-app-backend.default.svc.cluster.local
diff --git a/k8s/helmfile/env/local/tool-widar.values.yaml.gotmpl b/k8s/helmfile/env/local/tool-widar.values.yaml.gotmpl
@@ -1 +1,2 @@
-# the local service matches production, so this file is empty
+platform:
+  mediawikiBackendHost: mediawiki-139-app-backend.default.svc.cluster.local
diff --git a/k8s/helmfile/env/production/mediawiki-139.values.yaml.gotmpl b/k8s/helmfile/env/production/mediawiki-139.values.yaml.gotmpl
@@ -0,0 +1,75 @@
+image:
+  tag: "1.39-7.4-20230328-0"
+
+replicaCount:
+  backend: 1
+  web: 2
+  webapi: 2
+  alpha: 1
+
+mw:
+  redis:
+    # TODO fixme, no port injected into deployment...
+    # TODO fixme, no "database" ID injected into mediawiki
+    readServer: {{ .Values.services.redis.readHost }}
+    writeServer: {{ .Values.services.redis.writeHost }}
+    password:
+    passwordSecretName: redis-password
+    passwordSecretKey: password
+  elasticsearch:
+    host: elasticsearch-master.default.svc.cluster.local
+    port: 9200
+  mailgun:
+    enabled: false
+  platform:
+    apiBackendHost: api-app-backend.default.svc.cluster.local
+  settings:
+    allowedProxyCidr: "10.108.0.0/14"
+  db:
+    replica: sql-mariadb-secondary.default.svc.cluster.local
+    master: sql-mariadb-primary.default.svc.cluster.local
+  mail:
+    domain: "wikibase.cloud"
+  smtp:
+    enabled: true
+    smtpUserSecretName: smtp-credentials
+    smtpUserSecretKey: username
+    smtpPasswordSecretName: smtp-credentials
+    smtpPasswordSecretKey: password
+    host: smtp.eu.mailgun.org
+    port: 587
+  recaptcha:
+    sitekeySecretName: {{ .Values.external.recaptcha2.secretName }}
+    sitekeySecretKey: site_key
+    secretkeySecretName: {{ .Values.external.recaptcha2.secretName }}
+    secretkeySecretKey: secret_key
+
+resources:
+  web:
+    requests:
+      cpu: 150m
+      memory: 350Mi
+    limits:
+      cpu: 400m
+      memory: 750Mi
+  webapi:
+    requests:
+      cpu: 200m
+      memory: 250Mi
+    limits:
+      cpu: 500m
+      memory: 1200Mi
+  alpha:
+    requests:
+      cpu: 50m
+      memory: 40Mi
+    limits:
+      cpu: 500m
+      memory: 600Mi
+  backend:
+    requests:
+      cpu: 500m
+      memory: 600Mi
+    limits:
+      cpu: 1000m
+      memory: 1200Mi
diff --git a/k8s/helmfile/env/staging/api.values.yaml.gotmpl b/k8s/helmfile/env/staging/api.values.yaml.gotmpl
@@ -1,5 +1,5 @@
 image:
-  tag: "8x.10.1"
+  tag: "8x.11.0"
 
 ingress:
   tls:
@@ -8,11 +8,11 @@ ingress:
     secretName: wikibase-dev-tls
 
 platform:
-  backendMwHost: mediawiki-138-app-backend.default.svc.cluster.local
+  backendMwHost: mediawiki-139-app-backend.default.svc.cluster.local
 
 wbstack:
-  wikiDbProvisionVersion: mw1.38-wbs1
-  wikiDbUseVersion: mw1.38-wbs1
+  wikiDbProvisionVersion: mw1.39-wbs1
+  wikiDbUseVersion: mw1.39-wbs1
   monitoringEmail: "wb-cloud-monitoring+staging@wikimedia.de"
   elasticSearch:
     enabledByDefault: true

diff --git a/k8s/helmfile/env/staging/mediawiki-139.values.yaml.gotmpl b/k8s/helmfile/env/staging/mediawiki-139.values.yaml.gotmpl
@@ -0,0 +1,8 @@
+image:
+  tag: "1.39-7.4-20230328-0"
+
+mw:
+  settings:
+    allowedProxyCidr: "10.112.0.0/14"
+  mail:
+    domain: "wikibase.dev"
diff --git a/k8s/helmfile/env/staging/platform-nginx.nginx.conf b/k8s/helmfile/env/staging/platform-nginx.nginx.conf
@@ -7,7 +7,47 @@ map_hash_bucket_size 128;
 #WHERE deleted_at IS NULL
 #LIMIT 900
 map $host $mwversion {
-        default "138-app";
+        default "139-app";
+        "goathub.wikibase.dev" "138-app";
+        "deerbase.wikibase.dev" "138-app";
+        "coffeebase.wikibase.dev" "138-app";
+        "somethingwitty.wikibase.dev" "138-app";
+        "goat-136.wikibase.dev" "138-app";
+        "candy-collection.wikibase.dev" "138-app";
+        "potatobase.wikibase.dev" "138-app";
+        "potato.wikibase.dev" "138-app";
+        "asjhfkjsahdkjahs.wikibase.dev" "138-app";
+        "feddypropswiki.wikibase.dev" "138-app";
+        "addshore-alpha.wikibase.dev" "138-app";
+        "faketestdomain.hopto.org" "138-app";
+        "wikibasehost.ddns.net" "138-app";
+        "addshore-fake-cusstom-domain-wikibase.duckdns.org" "138-app";
+        "deerstack.wikibase.dev" "138-app";
+        "cacti.wikibase.dev" "138-app";
+        "rose-collection.wikibase.dev" "138-app";
+        "migratetestbase.ddns.net" "138-app";
+        "biodiversity.wikibase.dev" "138-app";
+        "toms-test-wikibase.duckdns.org" "138-app";
+        "potatowiki3.wikibase.dev" "138-app";
+        "charlie-test-1.wikibase.dev" "138-app";
+        "deerdeerdeer.wikibase.dev" "138-app";
+        "blob-wiki.wikibase.dev" "138-app";
+        "test-lm-02.wikibase.dev" "138-app";
+        "testwikibase-jan.wikibase.dev" "138-app";
+        "aflex.wikibase.dev" "138-app";
+        "bh20subset1.wikibase.dev" "138-app";
+        "biomarker.wikibase.dev" "138-app";
+        "testa.wikibase.dev" "138-app";
+        "carrotwiki.wikibase.dev" "138-app";
+        "sweetcornbase.wikibase.dev" "138-app";
+        "mywayordatway.wikibase.dev" "138-app";
+        "redbull.wikibase.dev" "138-app";
+        "asdfg.wikibase.dev" "138-app";
+        "asfasfasfasfasf.wikibase.dev" "138-app";
+        "wbdev.ledeniz.de" "138-app";
+        "dev-test-wbaas-custom-deer.duckdns.org" "138-app";
+        "andrew.wikibase.dev" "138-app";
+        "mw138test.wikibase.dev" "138-app";
 }
 # Figure out which group of backends we might want to send the request to based on uri
 # TODO add Special:EntityData???

diff --git a/k8s/helmfile/env/staging/tool-quickstatements.values.yaml.gotmpl b/k8s/helmfile/env/staging/tool-quickstatements.values.yaml.gotmpl
@@ -1,2 +1,2 @@
 platform:
-  mediawikiBackendHost: mediawiki-138-app-backend.default.svc.cluster.local
+  mediawikiBackendHost: mediawiki-139-app-backend.default.svc.cluster.local
diff --git a/k8s/helmfile/env/staging/tool-widar.values.yaml.gotmpl b/k8s/helmfile/env/staging/tool-widar.values.yaml.gotmpl
@@ -1,2 +1,2 @@
 platform:
-  mediawikiBackendHost: mediawiki-138-app-backend.default.svc.cluster.local
+  mediawikiBackendHost: mediawiki-139-app-backend.default.svc.cluster.local
diff --git a/k8s/helmfile/helmfile.yaml b/k8s/helmfile/helmfile.yaml
@@ -155,6 +155,13 @@ releases:
     version: 0.10.6
     <<: *default_release
 
+  - name: mediawiki-139
+    namespace: default
+    chart: wbstack/mediawiki
+    version: 0.10.6
+    installed: {{ ne .Environment.Name "production" | toYaml }}
+    <<: *default_release
+
   - name: queryservice-ui
     namespace: default
     chart: wbstack/queryservice-ui