Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pull changes for deploying to it-sre infra #3663

Merged
merged 17 commits into from
May 22, 2019
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion Jenkinsfile
@@ -1,4 +1,4 @@
@Library('github.com/mozmeao/jenkins-pipeline@20171123.1')
@Library('github.com/mozilla-it/jenkins-pipeline@20171123.1')
def config
def docker_image
def dc_name
Expand Down Expand Up @@ -112,3 +112,4 @@ conduit {
sh "bin/slack-notify.sh --status success --stage 'Docker image ready to deploy: ${docker_image}'"
}
}

4 changes: 2 additions & 2 deletions bin/slack-notify.sh
Expand Up @@ -5,9 +5,9 @@ set -eo pipefail
# BRANCH_NAME, BUILD_NUMBER

# defaults and constants
CHANNEL="sumodev"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we use the existing channel? It's already well known by users and it usually doesn't have any noise apart from dev related topics. The new channel has also logs from yar which will make it harder to locate the builds.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd be happy to but in the interim it's going to be noisy while we get everything stood up, and I don't want to confuse a non-production event with a production event. Could we perhaps switch back post-migration to the current channels?

CHANNEL="itsre-sumo"
PROJECT_NAME="kitsune"
BLUE_BUILD_URL="https://ci.vpn1.moz.works/blue/organizations/jenkins/Kitsune"
BLUE_BUILD_URL="https://ci.sumo.mozit.cloud"
BLUE_BUILD_URL="${BLUE_BUILD_URL}/detail/${BRANCH_NAME/\//%2f}/${BUILD_NUMBER}/pipeline"

# parse cli args
Expand Down
24 changes: 12 additions & 12 deletions docker-compose.yml
@@ -1,7 +1,7 @@
version: '3.4'
services:
web:
image: mozmeao/kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}
command: ./bin/run-dev.sh
env_file: .env
volumes:
Expand All @@ -15,7 +15,7 @@ services:
- "8000:8000"

test:
image: mozmeao/kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}
command: ./bin/run-unit-tests.sh
env_file: .env-test
volumes:
Expand All @@ -27,15 +27,15 @@ services:
- redis

test-js:
image: mozmeao/kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}
command: ./bin/run-mocha-tests.sh
env_file: .env-test
volumes:
- ./:/app:delegated
user: ${UID:-kitsune}

test-image:
image: mozmeao/kitsune:full-no-locales-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:full-no-locales-${GIT_COMMIT_SHORT:-latest}
command: ./bin/run-unit-tests.sh
env_file: .env-test
depends_on:
Expand All @@ -44,12 +44,12 @@ services:
- redis

test-image-js:
image: mozmeao/kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}
command: ./bin/run-mocha-tests.sh
env_file: .env-test

lint-l10n:
image: mozmeao/kitsune:base-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:base-${GIT_COMMIT_SHORT:-latest}
command: dennis-cmd lint --errorsonly locale
env_file: .env-test
volumes:
Expand All @@ -62,23 +62,23 @@ services:
target: base
args:
- GIT_SHA
image: mozmeao/kitsune:base-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:base-${GIT_COMMIT_SHORT:-latest}

base-dev:
build:
context: .
target: base-dev
args:
- GIT_SHA
image: mozmeao/kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:base-dev-${GIT_COMMIT_SHORT:-latest}

staticfiles:
build:
context: .
target: staticfiles
args:
- GIT_SHA
image: mozmeao/kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:staticfiles-${GIT_COMMIT_SHORT:-latest}

locales:
build:
Expand All @@ -87,7 +87,7 @@ services:
args:
- GIT_SHA
- LOCALE_ENV
image: mozmeao/kitsune:locales-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:locales-${GIT_COMMIT_SHORT:-latest}

full-no-locales:
build:
Expand All @@ -96,7 +96,7 @@ services:
args:
- GIT_SHA
- LOCALE_ENV
image: mozmeao/kitsune:full-no-locales-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:full-no-locales-${GIT_COMMIT_SHORT:-latest}

full:
build:
Expand All @@ -105,7 +105,7 @@ services:
args:
- GIT_SHA
- LOCALE_ENV
image: mozmeao/kitsune:full-${GIT_COMMIT_SHORT:-latest}
image: itsre/sumo-kitsune:full-${GIT_COMMIT_SHORT:-latest}

# EXTERNAL SERVICES
mariadb:
Expand Down
4 changes: 2 additions & 2 deletions docker/bin/build-docker-images.sh
@@ -1,7 +1,7 @@
#!/bin/bash
set -e

DOCKER_REPO=${DOCKER_REPO:-mozmeao/kitsune}
DOCKER_REPO=${DOCKER_REPO:-itsre/sumo-kitsune}
GIT_SHA=${GIT_SHA:-auto}
GIT_SHA_SHORT=${GIT_SHA_SHORT:-$GIT_SHA}
LOCALE_ENV=${LOCALE_ENV:-master}
Expand All @@ -18,7 +18,7 @@ do
-t ${DOCKER_REPO}:${image}-${GIT_SHA_SHORT} \
--cache-from ${DOCKER_REPO}:${image}-latest \
--cache-from kitsune:${image}-latest \
-f docker/dockerfiles/${image} \
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ziegeer could you please elaborate a bit about this change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this script is for building docker images locally, which I don't think anyone actually does anymore, but I had to and it pointed to some non-existant dockerfiles that look to have been previously consolidated into a single Dockerfile. So I just pointed the script at that and it worked.

-f Dockerfile \
--build-arg GIT_SHA=${GIT_SHA} \
--build-arg LOCALE_ENV=${LOCALE_ENV} .
done
2 changes: 1 addition & 1 deletion docker/bin/fetch-l10n-files.sh
Expand Up @@ -7,7 +7,7 @@ LOCALE_ENV="${LOCALE_ENV:-master}"
if [[ "$LOCALE_ENV" == "master" ]]; then
LOCALE_URL="https://github.com/mozilla-l10n/sumo-l10n/archive/master.tar.gz"
elif [[ "$LOCALE_ENV" == "production" ]]; then
LOCALE_URL="https://github.com/mozmeao/sumo-l10n-prod/archive/master.tar.gz"
LOCALE_URL="https://github.com/mozilla-it/sumo-l10n-prod/archive/master.tar.gz"
else
echo "Unknown value for LOCALE_ENV: $LOCALE_ENV"
exit 1
Expand Down
2 changes: 1 addition & 1 deletion docker/bin/pull-docker-images.sh
@@ -1,7 +1,7 @@
#!/bin/bash
set -e

DOCKER_REPO=${DOCKER_REPO:-mozmeao/kitsune}
DOCKER_REPO=${DOCKER_REPO:-itsre/sumo-kitsune}

for image in base base-dev staticfiles locales full-no-locales full;
do
Expand Down
2 changes: 1 addition & 1 deletion docker/bin/push-docker-images.sh
@@ -1,7 +1,7 @@
#!/bin/bash
set -e

DOCKER_REPO=${DOCKER_REPO:-mozmeao/kitsune}
DOCKER_REPO=${DOCKER_REPO:-itsre/sumo-kitsune}
GIT_SHA=${GIT_SHA:-latest}

if ! ([ "$DOCKER_USERNAME" ] || [ -f ~/.docker/config.json ]);
Expand Down
2 changes: 1 addition & 1 deletion docker/bin/set_git_env_vars.sh
Expand Up @@ -21,5 +21,5 @@ fi
export BRANCH_NAME_SAFE="${BRANCH_NAME/\//-}"
export BRANCH_AND_COMMIT="${BRANCH_NAME_SAFE}-${GIT_COMMIT}"
# Docker Hub Stuff
export DEPLOYMENT_DOCKER_REPO="mozmeao/kitsune"
export DEPLOYMENT_DOCKER_REPO="itsre/sumo-kitsune"
export DEPLOYMENT_DOCKER_IMAGE="${DEPLOYMENT_DOCKER_REPO}:full-${GIT_COMMIT_SHORT}"
8 changes: 3 additions & 5 deletions docker/bin/upload-staticfiles.sh
Expand Up @@ -2,7 +2,7 @@

set -exo pipefail

DOCKER_REPO=${DOCKER_REPO:-mozmeao/kitsune}
DOCKER_REPO=${DOCKER_REPO:-itsre/sumo-kitsune}
GIT_COMMIT=${GIT_COMMIT:-latest}
GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT:-$GIT_COMMIT}
CONTAINER_NAME="kitsune-static-${GIT_COMMIT}"
Expand All @@ -26,14 +26,12 @@ for BUCKET in stage prod; do
aws s3 sync \
--acl public-read \
--cache-control "max-age=315360000, public, immutable" \
--profile sumo-media \
"./${TMP_DIR_HASHED}" "s3://sumo-${BUCKET}-media/static/"
"./${TMP_DIR_HASHED}" "s3://mozit-sumo-${BUCKET}-media/static/"
# non-hashed-filenames
aws s3 sync \
--acl public-read \
--cache-control "max-age=21600, public" \
--profile sumo-media \
"./${TMP_DIR}" "s3://sumo-${BUCKET}-media/static/"
"./${TMP_DIR}" "s3://mozit-sumo-${BUCKET}-media/static/"
done

rm -rf "${TMP_DIR}"
Expand Down
12 changes: 5 additions & 7 deletions docs/kitsune-support.md
Expand Up @@ -4,8 +4,8 @@

High level:

- [SUMO Infra home](https://github.com/mozmeao/infra/tree/master/apps/sumo)
- [Deploying SUMO](https://github.com/mozilla/kitsune/tree/master/k8s#deploying-sumo)
- [SUMO Infra home](https://github.com/mozilla-it/sumo-infra)
- [Deploying SUMO](https://github.com/mozilla-it/kitsune/tree/master/k8s#deploying-sumo)
ziegeer marked this conversation as resolved.
Show resolved Hide resolved
- [MozMEAO escalation path](https://mana.mozilla.org/wiki/pages/viewpage.action?pageId=50267455)

- [Architecture diagram](https://raw.githubusercontent.com/mozilla/kitsune/master/docs/SUMO%20architecture%202018.png)
Expand All @@ -15,9 +15,7 @@ High level:
Tech details:

- [SUMO K8s deployments/services/secrets templates](https://github.com/mozilla/kitsune/tree/master/k8s/)
- [SUMO AWS resource definitions](https://github.com/mozmeao/infra/tree/master/apps/sumo/infra)
- [Shared resources (S3/Cloudfront)](https://github.com/mozmeao/infra/tree/master/apps/sumo/infra/shared)
- [per-region resources RDS/Redis](https://github.com/mozmeao/infra/tree/master/apps/sumo/infra/multi_region)
- [SUMO AWS resource definitions](https://github.com/mozilla-it/sumo-infra/tree/master/k8s/tf)



Expand Down Expand Up @@ -237,7 +235,7 @@ sumo-prod-web 50 50 50 50 331d
2. ensure you are in the `Oregon` region
3. search for and select the `EC2` service in the AWS console
4. select `Auto Scaling Groups` from the navigation on the left side of the page
5. click on the `nodes.oregon-a.moz.works` or `nodes.oregon-b.moz.works` row to select it
5. click on the `nodes.oregon-a.mozit.cloud` or `nodes.oregon-b.mozit.cloud` row to select it
6. from the `Actions` menu (close to the top of the page), click `Edit`
7. the `Details` tab for the ASG should appear, set the appropriate `Min`, `Desired` and `Max` values.
1. it's probably good to set `Min` and `Desired` to the same value in case the cluster autoscaler decides to scale down the cluster smaller than the `Min`.
Expand Down Expand Up @@ -303,4 +301,4 @@ There are limits that apply to using VPC ACLs documented [here](http://docs.aws.
kubectl -n sumo-prod scale --replicas=0 deployment/sumo-prod-cron
```
- **DNS**
- point the `prod-tp.sumo.moz.works` traffic policy at the Frankfurt ELB
- point the `prod-tp.sumo.mozit.cloud` traffic policy at the Frankfurt ELB
2 changes: 1 addition & 1 deletion jenkins.yml
@@ -1,3 +1,3 @@
project:
name: kitsune
docker_name: mozmeao/kitsune
docker_name: itsre/sumo-kitsune
2 changes: 1 addition & 1 deletion k8s/README.md
Expand Up @@ -134,7 +134,7 @@ Run basic acceptance tests with

`./acceptance-tests.sh <URL>`

E.g. `./acceptance-tests.sh https://dev.sumo.moz.works`
E.g. `./acceptance-tests.sh https://dev.sumo.mozit.cloud`


#### List of invoke available tasks
Expand Down
6 changes: 3 additions & 3 deletions k8s/commander.sh
Expand Up @@ -2,8 +2,8 @@
set -exo pipefail
GREEN='\033[1;32m'
NC='\033[0m' # No Color
SLACK_CHANNEL=sumodev
DOCKER_HUB="https://hub.docker.com/r/mozmeao/kitsune/tags/"
SLACK_CHANNEL=itsre-sumo
DOCKER_HUB="https://hub.docker.com/r/itsre/sumo-kitsune/tags/"


function whatsdeployed {
Expand Down Expand Up @@ -37,7 +37,7 @@ function deploy {
post-deploy $@

if command -v slack-cli > /dev/null; then
slack-cli -d "${SLACK_CHANNEL}" ":tada: Successfully deployed <${DOCKER_HUB}|full-${COMMIT_HASH}> to <https://${REGION_ENV}-${REGION}.sumo.moz.works/|SUMO-${REGION_ENV} in ${REGION}>"
slack-cli -d "${SLACK_CHANNEL}" ":tada: Successfully deployed <${DOCKER_HUB}|full-${COMMIT_HASH}> to <https://${REGION_ENV}-${REGION}.sumo.mozit.cloud/|SUMO-${REGION_ENV} in ${REGION}>"
fi
printf "${GREEN}OK${NC}\n"
}
Expand Down
12 changes: 6 additions & 6 deletions k8s/regions/frankfurt/dev.yaml
Expand Up @@ -5,7 +5,7 @@ kubernetes:
nodeport_name: "sumo-nodeport"
secrets_name: "sumo-secrets-dev"
image:
repo: "mozmeao/kitsune"
repo: "itsre/sumo-kitsune"
tag: "full-latest"
pull_policy: "Always"
# default values
Expand Down Expand Up @@ -47,7 +47,7 @@ kubernetes:
timeout_seconds: 5
app:
allowed_cidr_nets: SECRET
allowed_hosts: ".sumo.moz.works"
allowed_hosts: ".sumo.mozit.cloud,support-dev.allizom.org"
aws_access_key_id: SECRET
aws_s3_custom_domain: SECRET
aws_secret_access_key: SECRET
Expand Down Expand Up @@ -87,7 +87,7 @@ app:
dms_update_product_details: SECRET
dms_update_top_contributors: SECRET
dms_update_weekly_votes: SECRET
k8s_domain: "frankfurt.moz.works"
k8s_domain: "frankfurt.mozit.cloud"
email_host: SECRET
email_host_user: SECRET
email_host_password: SECRET
Expand All @@ -97,7 +97,7 @@ app:
email_use_tls: True
enable_admin: True
enable_whitenoise: True
enforce_host: dev.sumo.moz.works
enforce_host: dev.sumo.mozit.cloud
engage_robots: False
es_index_prefix: "sumo_dev"
es_http_auth: SECRET
Expand All @@ -108,7 +108,7 @@ app:
ga_key: SECRET
ga_profile_id: SECRET
gtm_container_id: "UA-36116321-2"
media_url: "https://dev-cdn.sumo.mozilla.net/"
media_url: "https://user-media-dev-cdn.itsre-sumo.mozilla.net/"
new_relic_app_name: "sumo-dev-frankfurt"
new_relic_license_key: SECRET
pipeline_enabled: True
Expand All @@ -121,7 +121,7 @@ app:
sentry_dsn: SECRET
session_cookie_secure: True
stage: True
static_url: "https://static-media-dev-cdn.sumo.mozilla.net/static/"
static_url: "https://static-media-dev-cdn.itsre-sumo.mozilla.net/static/"
statsd_client: SECRET
statsd_host: SECRET
statsd_prefix: "sumo-dev.frankfurt"
Expand Down
10 changes: 5 additions & 5 deletions k8s/regions/frankfurt/prod.yaml
Expand Up @@ -5,7 +5,7 @@ kubernetes:
nodeport_name: "sumo-nodeport"
secrets_name: "sumo-secrets-prod"
image:
repo: "mozmeao/kitsune"
repo: "itsre/sumo-kitsune"
tag: "full-latest"
pull_policy: "Always"
# default values
Expand Down Expand Up @@ -46,7 +46,7 @@ kubernetes:

app:
allowed_cidr_nets: SECRET
allowed_hosts: "prod.sumo.moz.works,prod-frankfurt.sumo.moz.works,support.mozilla.com,support.mozilla.org"
allowed_hosts: "prod.sumo.mozit.cloud,prod-frankfurt.sumo.mozit.cloud,support.mozilla.com,support.mozilla.org"
aws_access_key_id: SECRET
aws_s3_custom_domain: SECRET
aws_secret_access_key: SECRET
Expand Down Expand Up @@ -85,15 +85,15 @@ app:
dms_update_product_details: SECRET
dms_update_top_contributors: SECRET
dms_update_weekly_votes: SECRET
k8s_domain: "frankfurt.moz.works"
k8s_domain: "frankfurt.mozit.cloud"
email_host: SECRET
email_host_user: SECRET
email_host_password: SECRET
email_port: 25
email_use_tls: True
enable_admin: False
enable_whitenoise: True
enforce_host: support.mozilla.org,prod-frankfurt.sumo.moz.works
enforce_host: support.mozilla.org,prod-frankfurt.sumo.mozit.cloud
engage_robots: False
es_index_prefix: "sumo_prod"
es_http_auth: SECRET
Expand All @@ -104,7 +104,7 @@ app:
ga_key: SECRET
ga_profile_id: SECRET
gtm_container_id: "UA-36116321-2"
media_url: "https://prod-cdn.sumo.mozilla.net/"
media_url: "https://user-media-prod-cdn.itsre-sumo.mozilla.net/"
new_relic_app_name: "sumo-prod-frankfurt"
new_relic_license_key: SECRET
oidc_op_authorization_endpoint: SECRET
Expand Down