wireapp · zebot · Nov 3, 2022 · Nov 4, 2022 · Nov 4, 2022 · Nov 7, 2022
diff --git a/.envrc b/.envrc
@@ -5,7 +5,11 @@
 # or any of the `default.nix` files change. We do this by adding all these files
 # to the nix store and using the store paths as a cache key.
 
-store_paths=$(find . -name default.nix  | grep -v '^./nix' | grep -v '^./dist-newstyle' | xargs nix-store --add ./nix)
+nix_files=$(find . -name '*.nix' | grep -v '^./dist-newstyle')
+for nix_file in $nix_files; do
+  watch_file "$nix_file"
+done
+store_paths=$(echo "$nix_files" | xargs nix-store --add ./nix)
 layout_dir=$(direnv_layout_dir)
 env_dir=./.env
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -4,6 +4,25 @@ on:
    branches: [master, develop]
 
 jobs:
+  treefmt:
+   name: Run treefmt
+   environment: cachix # for secrets
+   runs-on: ubuntu-latest
+   steps:
+     - uses: actions/checkout@v2
+       with:
+         submodules: true
+     - uses: cachix/install-nix-action@v14.1
+     - uses: cachix/cachix-action@v10
+       with:
+         name: wire-server
+         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+     - name: Install treefmt and nixpkgs-fmt (without pulling in all of dev-env)
+       run: nix-env -if nix/default.nix -iA pkgs.treefmt pkgs.nixpkgs-fmt pkgs.shellcheck
+     - name: Run treefmt
+       run: treefmt
+
   build-docs:
     name: Build docs
     environment: cachix

diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,3 @@
-[submodule "services/nginz/third_party/headers-more-nginx-module"]
-	path = services/nginz/third_party/headers-more-nginx-module
-	url = https://github.com/openresty/headers-more-nginx-module.git 
-[submodule "services/nginz/third_party/nginx-module-vts"]
-	path = services/nginz/third_party/nginx-module-vts
-	url = https://github.com/vozlt/nginx-module-vts.git
 [submodule "libs/wire-message-proto-lens/generic-message-proto"]
 	path = libs/wire-message-proto-lens/generic-message-proto
 	url = https://github.com/wireapp/generic-message-proto
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,88 @@
+# [2022-11-29] (Chart Release 4.27.0)
+
+## Release notes
+
+
+* This realease migrates data from `galley.member_client` to `galley.mls_group_member_client`. When upgrading wire-server no manual steps are required. (#2859)
+
+* Upgrade webapp version to 2022-11-28-production.0-v0.31.9-0-c1cde9b (#2302)
+
+
+## API changes
+
+
+* Added global conversation type and GET endpoint (`GET /teams/:tid/conversations/global`). (#2753)
+
+* Support MLS self-conversations via a new endpoint `GET /conversations/mls-self`. This removes the `PUT` counterpart introduced in #2730 (#2839)
+
+* List the MLS self-conversation automatically without needing to call `GET /conversations/mls-self` first (#2856)
+
+* Support MLS self-conversations via a new endpoint `PUT /conversations/mls-self` (#2730)
+
+
+## Features
+
+
+* A team member's role can now be provisioned via SCIM (#2851, #2855)
+
+
+## Bug fixes and other updates
+
+
+* Avoid client deletion edge case condition which can lead to inconsistent data between brig and galley's clients tables. (#2830)
+
+* Do not list MLS self-conversation in client API v1 and v2 if it exists (#2872)
+
+* Prevention of storing unnecessary data in the database if adding a bot to a conversation fails. (#2870)
+
+* Fix bug in MLS user removal from conversation: the list of removed clients has to be compared with those in the conversation, not the list of *all* clients of that user (#2817)
+
+* For sftd/coturn/restund, fixed a bug in external ip address lookup, in case Kubernetes Node Name doesn't equal hostname. (#2837)
+
+* Requesting a new token with the client_id now works correctly when the old token is part of the request (#2860)
+
+
+## Internal changes
+
+
+* Add tests for invitation urls in team invitation responses. These depend on the settings of galley. (#2797)
+
+* Remove support for compiling local docker images with buildah. Nix is used to build docker images these days (#2822)
+
+* bump nginx-module-vts from v0.1.15 to v0.2.1 (#2827)
+
+* Nix-created docker images: add some debugging tools in the containers, and add 'make build-image-<packagename>' for convenience (#2829)
+
+* Split galley API routes and handler definitions into several modules (#2820)
+
+* Default intraListing to true. This means that the list of clients, so far saved in both brig's and galley's databases, will still be written to both, but only read from brig's database. This avoids cases where these two tables go out of sync. Brig becomes the source of truth for clients. In the future, if this holds, code and data for galley's clients table can be removed. (#2847)
+
+* Build nginz and nginz_disco docker images using nix (#2796)
+
+* Bump nixpkgs to latest unstable. Stop using forked nixpkgs. (#2828)
+
+* Brig calling API is now migrated to servant (#2815)
+
+* Fixed flaky feature TTL integration test (#2823)
+
+* Brig teams API is now migrated to servant (#2824)
+
+* Backoffice Swagger 2.x docs is exposed on `/` and the old Swagger has been removed. Backoffice helm chart only runs stern without an extra nginx. (#2846)
+
+* Stern API endpoint `GET ejpd-info` has now the correct HTTP method (#2850)
+
+* External commits: add additional checks (#2852)
+
+* Golden tests for conversation and feature config event schemas (#2861)
+
+* Refactor and simplify MLS message handling logic (#2844)
+
+* Replay external backend proposals after forwarding external commits.
+  One column added to Galley's mls_proposal_refs. (#2842)
+
+* Use treefmt to ensure consistent formatting of .nix files, use for shellcheck too (#2831)
+
+
 # [2022-11-03] (Chart Release 4.26.0)
 
 ## Release notes
@@ -63,8 +148,6 @@
 
 * Convert brig's auth endpoints to servant (#2750)
 
-* bump nginx-module-vts from v0.1.15 to v0.2.1 (#2793)
-
 * Remove deprecated table for storing scim external_ids.
 
   Data has been migrated away in [release 2021-03-21 (Chart Release 2.103.0)](https://github.com/wireapp/wire-server/releases/tag/v2021-03-21) (see `/services/spar/migrate-data/src/Spar/DataMigration/V1_ExternalIds.hs`); last time it has been touched in production is before upgrade to [release 2021-03-23 (Chart Release 2.104.0)](https://github.com/wireapp/wire-server/releases/tag/v2021-03-23). (#2768)

diff --git a/Makefile b/Makefile
@@ -14,9 +14,7 @@ CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
 CHARTS_RELEASE        := wire-server redis-ephemeral redis-cluster databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress  fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn inbucket
-BUILDAH_PUSH          ?= 0
 KIND_CLUSTER_NAME     := wire-server
-BUILDAH_KIND_LOAD     ?= 1
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema
@@ -41,7 +39,6 @@ init:
 # Build all Haskell services and executables, run unit tests
 .PHONY: install
 install: init
-	cabal update
 	cabal build all
 	./hack/bin/cabal-run-all-tests.sh
 	./hack/bin/cabal-install-artefacts.sh all
@@ -110,12 +107,16 @@ ghcid:
 
 # Used by CI
 .PHONY: lint-all
-lint-all: formatc hlint-check-all shellcheck check-local-nix-derivations
+lint-all: formatc hlint-check-all check-local-nix-derivations treefmt
 
 .PHONY: hlint-check-all
 hlint-check-all:
 	./tools/hlint.sh -f all -m check
 
+.PHONY: hlint-inplace-all
+hlint-inplace-all:
+	./tools/hlint.sh -f all -m inplace
+
 .PHONY: hlint-check-pr
 hlint-check-pr:
 	./tools/hlint.sh -f pr -m check
@@ -124,11 +125,6 @@ hlint-check-pr:
 hlint-inplace-pr:
 	./tools/hlint.sh -f pr -m inplace
 
-
-.PHONY: hlint-inplace-all
-hlint-inplace-all:
-	./tools/hlint.sh -f all -m inplace
-
 .PHONY: hlint-check
 hlint-check:
 	./tools/hlint.sh -f changeset -m check
@@ -156,7 +152,12 @@ format:
 # formats all Haskell files even if local changes are not committed to git
 .PHONY: formatf
 formatf:
-	./tools/ormolu.sh -f
+	./tools/ormolu.sh -f pr
+
+# formats all Haskell files even if local changes are not committed to git
+.PHONY: formatf-all
+formatf-all:
+	./tools/ormolu.sh -f all
 
 # checks that all Haskell files are formatted; fail if a `make format` run is needed.
 .PHONY: formatc
@@ -173,13 +174,23 @@ add-license:
 	@echo ""
 	@echo "you might want to run 'make formatf' now to make sure ormolu is happy"
 
-.PHONY: shellcheck
-shellcheck:
-	./hack/bin/shellcheck.sh
+.PHONY: treefmt
+treefmt:
+	treefmt
 
 #################################
 ## docker targets
 
+.PHONY: build-image-%
+build-image-%:
+	nix-build ./nix -A wireServer.imagesNoDocs.$(*) && \
+	./result | docker load | tee /tmp/imageName-$(*) && \
+	imageName=$$(grep quay.io /tmp/imageName-$(*) | awk '{print $$3}') && \
+	echo 'You can run your image locally using' && \
+	echo "  docker run -it --entrypoint bash $$imageName" && \
+	echo 'or upload it using' && \
+	echo "  docker push $$imageName"
+
 .PHONY: upload-images
 upload-images:
 	./hack/bin/upload-images.sh imagesNoDocs
@@ -200,7 +211,10 @@ git-add-cassandra-schema: db-reset git-add-cassandra-schema-impl
 .PHONY: git-add-cassandra-schema-impl
 git-add-cassandra-schema-impl:
 	$(eval CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1'))
-	( echo '-- automatically generated with `make git-add-cassandra-schema`' ; docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) > ./cassandra-schema.cql
+	( echo '-- automatically generated with `make git-add-cassandra-schema`'; \
+      docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) \
+    | sed "s/CREATE TABLE galley_test.member_client/-- NOTE: this table is unused. It was replaced by mls_group_member_client\nCREATE TABLE galley_test.member_client/g" \
+      > ./cassandra-schema.cql
 	git add ./cassandra-schema.cql
 
 .PHONY: cqlsh
@@ -409,24 +423,6 @@ upload-charts: charts-release
 echo-release-charts:
 	@echo ${CHARTS_RELEASE}
 
-.PHONY: buildah-docker
-buildah-docker: buildah-docker-nginz
-	./hack/bin/buildah-compile.sh all
-	BUILDAH_PUSH=${BUILDAH_PUSH} KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD}  ./hack/bin/buildah-make-images.sh
-
-.PHONY: buildah-docker-nginz
-buildah-docker-nginz:
-	BUILDAH_PUSH=${BUILDAH_PUSH} KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD}  ./hack/bin/buildah-make-images-nginz.sh
-
-.PHONY: buildah-docker-%
-buildah-docker-%:
-	./hack/bin/buildah-compile.sh $(*)
-	BUILDAH_PUSH=${BUILDAH_PUSH} EXECUTABLES=$(*) KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME} BUILDAH_KIND_LOAD=${BUILDAH_KIND_LOAD} ./hack/bin/buildah-make-images.sh
-
-.PHONY: buildah-clean
-buildah-clean:
-	./hack/bin/buildah-clean.sh
-
 .PHONY: kind-cluster
 kind-cluster:
 	kind create cluster --name $(KIND_CLUSTER_NAME)

diff --git a/cassandra-schema.cql b/cassandra-schema.cql
@@ -364,6 +364,7 @@ CREATE TABLE galley_test.group_id_conv_id (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+-- NOTE: this table is unused. It was replaced by mls_group_member_client
 CREATE TABLE galley_test.member_client (
     conv uuid,
     user_domain text,
@@ -430,6 +431,29 @@ CREATE TABLE galley_test.conversation_codes (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE galley_test.mls_group_member_client (
+    group_id blob,
+    user_domain text,
+    user uuid,
+    client text,
+    key_package_ref blob,
+    PRIMARY KEY (group_id, user_domain, user, client)
+) WITH CLUSTERING ORDER BY (user_domain ASC, user ASC, client ASC)
+    AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE galley_test.clients (
     user uuid PRIMARY KEY,
     clients set<text>
@@ -550,6 +574,7 @@ CREATE TABLE galley_test.mls_proposal_refs (
     group_id blob,
     epoch bigint,
     ref blob,
+    origin int,
     proposal blob,
     PRIMARY KEY (group_id, epoch, ref)
 ) WITH CLUSTERING ORDER BY (epoch ASC, ref ASC)

diff --git a/changelog.d/5-internal/bump-nginx-module-vts b/changelog.d/5-internal/bump-nginx-module-vts