Release 2018-10-25

.gitignore

@@ -40,13 +40,22 @@ tools/api-simulations/reports
 .DS_Store
 brig-schema.yaml
 brig.yaml
+brig.integration-aws.yaml
 cannon.yaml
+cannon.integration-aws.yaml
 cargohold.yaml
+cargohold.integration-aws.yaml
 galley-schema.yaml
 galley.yaml
+galley.integration-aws.yaml
 gundeck-schema.yaml
 gundeck.yaml
+gundeck.integration-aws.yaml
 proxy.yaml
+proxy.integration-aws.yaml
+spar.yaml
+spar.integration-aws.yaml
+integration-aws.yaml
 DOCKER_ID*
 swagger-ui
 services/spar/spar.cabal

CHANGELOG.md

@@ -1,36 +1,58 @@
-# 2018-10-04
+# 2018-10-25  #500
 
-See the [pull request][#477] for the exact changeset.
+## New Features
+
+  * SSO: team member deletion, team deletion do not require
+    the user to have chosen a password.  (Needed for
+    SAML-authenticated team co-admins.)  #497
+
+  * SSO: `sso-initiate-bind` end-point for inviting ("binding")
+    existing users to SAML auth.  #496
+
+  * SSO: shell script for registering IdPs in wire-teams.
+    (`/deploy/services-demo/register_idp.sh`)  #489
+
+  * Allow setting a different endpoint for generating download links.
+    #480
+
+  * Allow setting specific ports for SMTP and use different image for
+    SMTP.  #481
+
+  * Route calls/config in the demo to brig.  #487
+
+## Internal Changes
+
+  * Metrics for spar (service for SSO).  #498
+
+  * Upgrade to stackage lts-11.  #478
+
+  * Upgrade cql-io library.  #495
+
+  * Allow easily running tests against AWS.  #482
+
+
+# 2018-10-04  #477
 
 ## Highlights
 
-  * [#469][] We now store the `otr_muted_status` field per conversation,
+  * We now store the `otr_muted_status` field per conversation,
     suitable for supporting more notifications options than just "muted/not
-    muted". The exact meaning of this field is client-dependent.
+    muted". The exact meaning of this field is client-dependent.  #469
 
-  * [#467][] Our schema migration tools (which you are probably using if
+  * Our schema migration tools (which you are probably using if
     you're doing self-hosting) are more resilient now. They have longer
     timeouts and they wait for schema consistency across peers before
-    reporting success.
+    reporting success.  #467
 
 ## Other changes
 
-  * [#474][] Building from scratch on macOS is now a tiny bit easier.
+  * Building from scratch on macOS is now a tiny bit easier.  #474
 
   * Various Spar fixes, breaking changes, refactorings, and what-not. Please
-    refer to the commit log, in particular commits [c173f42b][] and
-    [80d06c9a][].
+    refer to the commit log, in particular commits c173f42b and
+    80d06c9a.
 
   * Spar now only accepts a [subset][TLS ciphersuite] of available TLS
-    ciphers. See [af8299d4][].
-
-[#467]: https://github.com/wireapp/wire-server/pull/467
-[#469]: https://github.com/wireapp/wire-server/pull/469
-[#474]: https://github.com/wireapp/wire-server/pull/474
-[#477]: https://github.com/wireapp/wire-server/pull/477
-
-[80d06c9a]: https://github.com/wireapp/wire-server/commit/80d06c9aba9f8f6a36bb19f0963636504d403761
-[c173f42b]: https://github.com/wireapp/wire-server/commit/c173f42b570fd458553ab354099926d1bd841e4d
-[af8299d4]: https://github.com/wireapp/wire-server/pull/466/commits/af8299d4aada7485ecc15cf6e8fb200b46d83d74
+    ciphers. See af8299d4.
 
 [TLS ciphersuite]: https://hackage.haskell.org/package/tls-1.4.1/docs/src/Network-TLS-Extra-Cipher.html#ciphersuite_default

Makefile

@@ -1,4 +1,5 @@
-LANG := en_US.UTF-8
+SHELL            := /usr/bin/env bash
+LANG             := en_US.UTF-8
 HASKELL_SERVICES := proxy cannon cargohold brig galley gundeck
 SERVICES         := $(HASKELL_SERVICES) nginz
 DOCKER_USER      ?= wireserver
@@ -38,6 +39,16 @@ integration: fast
 	$(MAKE) -C services/gundeck i-fake-aws
 	$(MAKE) -C services/spar i
 
+.PHONY: integration-aws
+integration-aws: fast
+	# We run "i" instead of "integration" to avoid useless rebuilds
+	# (since after "fast" everything will be built already)
+	$(MAKE) -C services/cargohold i-aws
+	$(MAKE) -C services/galley i-aws
+	$(MAKE) -C services/brig i-aws
+	$(MAKE) -C services/gundeck i-aws
+	$(MAKE) -C services/spar i-aws
+
 .PHONY: haddock
 haddock:
 	WIRE_STACK_OPTIONS="--haddock --haddock-internal" make fast
@@ -89,10 +100,8 @@ DOCKER_DEV_VOLUMES := -v `pwd`:/src/wire-server
 DOCKER_DEV_IMAGE   := quay.io/wire/alpine-builder:local
 .PHONY: run-docker-builder
 run-docker-builder:
-	docker run -it $(DOCKER_DEV_NETWORK) $(DOCKER_DEV_VOLUMES) --rm $(DOCKER_DEV_IMAGE) /bin/bash || \
-	( echo "$(DOCKER_DEV_IMAGE) not found.  building locally.  hit ^C to interrupt." && \
-	  make -C build/alpine builder && \
-	  make $@ )
+	@echo "if this does not work, consider 'docker pull', 'docker tag', or 'make -C build-alpine builder'."
+	docker run -it $(DOCKER_DEV_NETWORK) $(DOCKER_DEV_VOLUMES) --rm $(DOCKER_DEV_IMAGE) /bin/bash
 
 #################################
 ## dependencies

build/alpine/Dockerfile.builder

@@ -19,6 +19,15 @@ RUN apk add --no-cache git ncurses && \
     git clone -b develop https://github.com/wireapp/wire-server.git && \
     cd wire-server && \
     stack update && \
-    echo -e "allow-different-user: true\n" >> /root/.stack/config.yaml && \
+    echo "allow-different-user: true"                                       >> /root/.stack/config.yaml && \
+    echo                                                                    >> /root/.stack/config.yaml && \
+    echo '# NB: do not touch following line!'                               >> /root/.stack/config.yaml && \
+    echo '# this image is used both for building docker images with the'    >> /root/.stack/config.yaml && \
+    echo '# integration tests (so they can be run on the ci) and for'       >> /root/.stack/config.yaml && \
+    echo '# interactive integration testing (with the working copy of the'  >> /root/.stack/config.yaml && \
+    echo '# host system mounted into the docker container).  in the latter' >> /root/.stack/config.yaml && \
+    echo '# use case, we want the docker container to write to its own'     >> /root/.stack/config.yaml && \
+    echo '# stack-work directory and not pollute the one on the host.'      >> /root/.stack/config.yaml && \
+    echo 'work-dir: .stack-docker'                                          >> /root/.stack/config.yaml && \
     stack --work-dir .stack-docker-profile build --pedantic --haddock --test --dependencies-only --no-run-tests --profile && \
     stack --work-dir .stack-docker         build --pedantic --haddock --test --dependencies-only --no-run-tests

deploy/docker-ephemeral/db-migrate/brig-index.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+until_ready() {
+    until $1; do echo 'service not ready yet'; sleep 5; done
+}
+
+until_ready "brig-index reset --elasticsearch-server http://elasticsearch:9200"

deploy/docker-ephemeral/db-migrate/brig-schema.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+until_ready() {
+    until $1; do echo 'service not ready yet'; sleep 5; done
+}
+
+until_ready "brig-schema --host cassandra --keyspace brig_test --replication-factor 1"

deploy/docker-ephemeral/db-migrate/galley-schema.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+until_ready() {
+    until $1; do echo 'service not ready yet'; sleep 5; done
+}
+
+until_ready "galley-schema --host cassandra --keyspace galley_test --replication-factor 1"

deploy/docker-ephemeral/db-migrate/gundeck-schema.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+until_ready() {
+    until $1; do echo 'service not ready yet'; sleep 5; done
+}
+
+until_ready "gundeck-schema --host cassandra --keyspace gundeck_test --replication-factor 1"

deploy/docker-ephemeral/db-migrate/spar-schema.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+until_ready() {
+    until $1; do echo 'service not ready yet'; sleep 5; done
+}
+
+until_ready "spar-schema --host cassandra --keyspace spar_test --replication-factor 1"

deploy/docker-ephemeral/docker-compose.yaml

@@ -21,17 +21,10 @@ services:
       - SERVICES=ses,sns
 
   basic_smtp: # needed for demo setup
-    # https://github.com/catatnight/docker-postfix
-    image: catatnight/postfix
+    # https://github.com/namshi/docker-smtp
+    image: namshi/smtp
     ports:
-        # if binding to port 25 is a problem,
-        # smtp connection handling needs to be made more generic to accept
-        # alternative ports. See changes introduced in
-        # https://github.com/wireapp/wire-server/pull/405
-      - "127.0.0.1:25:25"
-    environment:
-      - maildomain=mail.wiredemo.example.com
-      - smtp_user=dummy:dummy-smtp-password
+        - 127.0.0.1:2500:25
 
   fake_s3:
     image: minio/minio:RELEASE.2018-05-25T19-49-13Z
@@ -65,16 +58,54 @@ services:
     ports:
       - "127.0.0.1:9042:9042"
 
-  db_migrations:
-    image: quay.io/wire/migrations
+  db_migrations_brig_schema:
+    image: quay.io/wire/brig-schema
     depends_on:
-      - elasticsearch
       - cassandra
-    command: /scripts/db-migrate.sh
+    entrypoint: /scripts/db-migrate/brig-schema.sh
     volumes:
       - ./:/scripts
     links:
+      - cassandra
+
+  db_migrations_brig_index:
+    image: quay.io/wire/brig-index
+    depends_on:
       - elasticsearch
+    entrypoint: /scripts/db-migrate/brig-index.sh
+    volumes:
+      - ./:/scripts
+    links:
+      - elasticsearch
+
+  db_migrations_galley:
+    image: quay.io/wire/galley-schema
+    depends_on:
+      - cassandra
+    entrypoint: /scripts/db-migrate/galley-schema.sh
+    volumes:
+      - ./:/scripts
+    links:
+      - cassandra
+
+  db_migrations_gundeck:
+    image: quay.io/wire/gundeck-schema
+    depends_on:
+      - cassandra
+    entrypoint: /scripts/db-migrate/gundeck-schema.sh
+    volumes:
+      - ./:/scripts
+    links:
+      - cassandra
+
+  db_migrations_spar:
+    image: quay.io/wire/spar-schema
+    depends_on:
+      - cassandra
+    entrypoint: /scripts/db-migrate/spar-schema.sh
+    volumes:
+      - ./:/scripts
+    links:
       - cassandra
 
   aws_cli:

deploy/docker-ephemeral/run.sh

@@ -10,7 +10,13 @@ DOCKER_FILE="$SCRIPT_DIR/docker-compose.yaml"
 #
 #     brig: Schema Version too old! Expecting at least: 49, but got: 48
 #
-# So we always pull the migrations image first.
-docker pull quay.io/wire/migrations
+# So we always pull these migration images first.
+docker pull quay.io/wire/brig-schema
+docker pull quay.io/wire/galley-schema
+docker pull quay.io/wire/gundeck-schema
+docker pull quay.io/wire/spar-schema
+
+# elasticsearch does not do migrations, so the following line is not needed.
+#docker pull quay.io/wire/brig-index
 
 docker-compose --file "$DOCKER_FILE" up

deploy/services-demo/README.md

@@ -5,7 +5,8 @@ This document assumes that you have already compiled all services (i.e., you rea
 Use 2 different terminals and run:
 
 ```
-# On terminal 1, start the dependencies
+# On terminal 1, start the dependencies. Note that you should turn up the max memory
+# limit of docker. More on https://github.com/wireapp/wire-server/issues/326
 deploy/docker-ephemeral/run.sh
 ```
 
@@ -37,7 +38,7 @@ resources                            <- folder which contains secrets or other r
 
 ### Why do you describe this as a _demo_?
 
-* **no optimal performance; not highly-available**: The way that the data stores used are set up is done in a simple way that is not advisable for a production environment (e.g., cassandra uses a single node and Docker will manage the storage of your database data by writing the database files to disk on the host system using its own internal volume management). 
+* **no optimal performance; not highly-available**: The way that the data stores used are set up is done in a simple way that is not advisable for a production environment (e.g., cassandra uses a single node and Docker will manage the storage of your database data by writing the database files to disk on the host system using its own internal volume management).
 * **missing functionality**: Some other dependencies (such as the "fake" AWS services) do not provide the full functionality of the real AWS services (for instance, the fake SES doesn't actually send emails) nor do they have the same reliability and availability.
 * :warning: **insecure by default** :warning: :
     * **no private network**: Not only is `nginz` reachable on port 8080 from the outside world, but all other services and databases are also reachable from localhost, which, if you run this from e.g. your laptop, allows any other concurrently running process (or exploits thereof) to

deploy/services-demo/conf/brig.demo.yaml

@@ -35,9 +35,9 @@ internalEvents:
 
 emailSMS:
   email:
-    smtpEndpoint: 127.0.0.1
-    smtpUsername: dummy
-    smtpPassword: resources/smtp-secret.txt
+    smtpEndpoint:
+      host: 127.0.0.1
+      port: 2500
     smtpConnType: plain
   general:
     templateDir: resources/templates

deploy/services-demo/conf/nginz/nginx.conf

@@ -182,6 +182,10 @@ http {
       proxy_pass http://brig;
     }
 
+    location /calls/config {
+      include common_response_with_zauth.conf;
+      proxy_pass http://brig;
+    }
     # Cargohold Endpoints
 
     rewrite ^/api-docs/assets  /assets/api-docs?base_url=http://127.0.0.1:8080/ break;
@@ -325,6 +329,11 @@ http {
       proxy_pass http://spar;
     }
 
+    location /sso-initiate-bind {
+      include common_response_with_zauth.conf;
+      proxy_pass http://spar;
+    }
+
     location /identity-providers {
       include common_response_with_zauth.conf;
       proxy_pass http://spar;