Skip to content

Commit

Permalink
[shopsys] deployment (#2648)
Browse files Browse the repository at this point in the history
  • Loading branch information
grossmannmartin committed Jul 17, 2023
2 parents 38507d7 + 7f8d3f6 commit e2853e7
Show file tree
Hide file tree
Showing 13 changed files with 232 additions and 9 deletions.
27 changes: 21 additions & 6 deletions .gitlab-ci.yml
Expand Up @@ -7,6 +7,8 @@ stages:

.only-default: &only-default
rules:
- if: '$MIRROR_ONLY == "true"'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
when: never
Expand All @@ -22,6 +24,9 @@ variables:
before_script:
- echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} ${CI_REGISTRY} --password-stdin

# Can be removed if mirroring is not needed
include: gitlab/mirror-repository/gitlab-stage.yml

build:
stage: build
<<: *only-default
Expand All @@ -30,7 +35,7 @@ build:
script:
- cp -R ./docker/nginx/ ./app/docker/nginx/
- cd ./app/
- docker build -f ./docker/php-fpm/Dockerfile --target production --compress -t ${TAG} .
- docker build -f ./docker/php-fpm/Dockerfile --target production-project --compress -t ${TAG} .
- docker push ${TAG}
interruptible: true

Expand Down Expand Up @@ -67,13 +72,16 @@ test:storefront-standards-with-codegen:
needs:
- build-storefront
before_script:
- corepack enable
- corepack prepare --activate pnpm@8.6.0
- pnpm config set store-dir .pnpm-store
- cp ./app/schema.graphql ./storefront/schema.graphql
- cd ./storefront
- npm ci
script:
- pnpm install --frozen-lockfile
- ./check-next-public-variable.sh
- ./check-code-gen.sh
- npm run check
- pnpm run check
interruptible: true

tests:functional-and-acceptance:
Expand Down Expand Up @@ -142,7 +150,6 @@ review:
- chmod +x ./gitlab/scripts/set-urls-for-review.sh && ./gitlab/scripts/set-urls-for-review.sh setDomainsToDockerCompose
- docker-compose down --rmi all -v --remove-orphans
- docker-compose up -d
- "docker-compose exec -T php-fpm sed -i \"s/database_name:.*/database_name: ${COMPOSE_PROJECT_NAME}/\" config/parameters.yaml"
- ./gitlab/scripts/set-urls-for-review.sh setUrlsToDomainsUrls
- docker-compose exec -T php-fpm php phing -D production.confirm.action=y clean db-create build-review-ci
- ./gitlab/scripts/set-urls-for-review.sh printDomains
Expand All @@ -165,6 +172,8 @@ review:stop:
- docker-compose down --rmi all -v --remove-orphans
- /home/ci/clean.sh ${COMPOSE_PROJECT_NAME}
rules:
- if: '$MIRROR_ONLY == "true"'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: manual
allow_failure: true
Expand Down Expand Up @@ -217,14 +226,16 @@ service:merge_request:

.deploy: &deploy
image:
name: registry.shopsys.cz/devops/kubernetes-buildpack:1.1
name: shopsys/kubernetes-buildpack:0.9
stage: deploy
needs:
- build
- build-storefront
tags:
- deploy
rules:
- if: '$MIRROR_ONLY == "true"'
when: never
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
script:
Expand All @@ -240,6 +251,8 @@ deploy:production:
variables:
KUBE_CONFIG: ${KUBE_CONFIG_PROD}
rules:
- if: '$MIRROR_ONLY == "true"'
when: never
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
Expand All @@ -259,6 +272,8 @@ deploy:devel:
variables:
KUBE_CONFIG: ${KUBE_CONFIG_DEVEL}
rules:
- if: '$MIRROR_ONLY == "true"'
when: never
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- if: '$CI_COMMIT_BRANCH == "devel" || $CI_COMMIT_BRANCH =~ /^devel-.*$/'
Expand All @@ -271,7 +286,7 @@ deploy:devel:
test:gatling:
stage: test
image:
name: registry.shopsys.cz/devops/kubernetes-buildpack:1.1
name: shopsys/kubernetes-buildpack:0.9
tags:
- tests
variables:
Expand Down
7 changes: 7 additions & 0 deletions app/app/preload.php
@@ -0,0 +1,7 @@
<?php

declare(strict_types=1);

if (file_exists(dirname(__DIR__) . '/var/cache/prod/App_KernelProdContainer.preload.php')) {
require dirname(__DIR__) . '/var/cache/prod/App_KernelProdContainer.preload.php';
}
1 change: 1 addition & 0 deletions app/composer.json
Expand Up @@ -75,6 +75,7 @@
"scheb/2fa-qr-code": "^5.7",
"sensio/framework-extra-bundle": "^5.2",
"sentry/sentry-symfony": "^4.2.8",
"shopsys/deployment": "1.0.0",
"shopsys/form-types-bundle": "13.0.x-dev",
"shopsys/framework": "13.0.x-dev",
"shopsys/frontend-api": "13.0.x-dev",
Expand Down
2 changes: 1 addition & 1 deletion app/config/bundles.php
Expand Up @@ -34,7 +34,7 @@
Symfony\Bundle\WebProfilerBundle\WebProfilerBundle::class => ['dev' => true],
Symfony\Bundle\DebugBundle\DebugBundle::class => ['dev' => true],
Symfony\WebpackEncoreBundle\WebpackEncoreBundle::class => ['all' => true],
Shopsys\S3Bridge\S3BridgeBundle::class => ['all' => true],
Shopsys\S3Bridge\ShopsysS3BridgeBundle::class => ['all' => true],
Scheb\TwoFactorBundle\SchebTwoFactorBundle::class => ['all' => true],
Sentry\SentryBundle\SentryBundle::class => ['prod' => true],
Overblog\DataLoaderBundle\OverblogDataLoaderBundle::class => ['all' => true],
Expand Down
5 changes: 5 additions & 0 deletions app/config/packages/frontend_api.yaml
@@ -0,0 +1,5 @@
parameters:
shopsys.frontend_api.keys_filepath: '%kernel.project_dir%/config/frontend-api'
shopsys.frontend_api.domains:
- 1
- 2
1 change: 1 addition & 0 deletions app/deploy/basicHttpAuth
@@ -0,0 +1 @@
shopsys:$apr1$EwnomQEd$xxn68cByy.RTc/iplRNms/
124 changes: 124 additions & 0 deletions app/deploy/deploy-project.sh
@@ -0,0 +1,124 @@
#!/bin/bash -e

BASE_PATH="$(realpath "$(dirname "$0")/..")"
CONFIGURATION_TARGET_PATH="${BASE_PATH}/var/deployment/kubernetes"
BASIC_AUTH_PATH="${BASE_PATH}/deploy/basicHttpAuth"
DEPLOY_TARGET_PATH="${BASE_PATH}/var/deployment/deploy"
CI_ENVIRONMENT_SLUG=${CI_ENVIRONMENT_SLUG}
SENTRY_DSN=${SENTRY_DSN}

function deploy() {
DOMAINS=(
DOMAIN_HOSTNAME_1
DOMAIN_HOSTNAME_2
)

ENABLE_AUTOSCALING=true

declare -A PARAMETERS=(
["parameters.trusted_proxies[+]"]=10.0.0.0/8
)

declare -A ENVIRONMENT_VARIABLES=(
["APP_SECRET"]=${APP_SECRET}
["DATABASE_HOST"]=${POSTGRES_DATABASE_IP_ADDRESS}
["DATABASE_PORT"]=${POSTGRES_DATABASE_PORT}
["DATABASE_NAME"]=${PROJECT_NAME}
["DATABASE_USER"]=${PROJECT_NAME}
["DATABASE_PASSWORD"]=${POSTGRES_DATABASE_PASSWORD}
["ELASTICSEARCH_HOST"]=${ELASTICSEARCH_URLS}
["ELASTIC_SEARCH_INDEX_PREFIX"]=${PROJECT_NAME}
["REDIS_PREFIX"]=${PROJECT_NAME}
["MAILER_DSN"]=${MAILER_DSN}

["GOPAY_IS_PRODUCTION_MODE"]=${GOPAY_IS_PRODUCTION_MODE}
["GOPAY_EN_GOID"]=${GOPAY_EN_GOID}
["GOPAY_EN_CLIENTID"]=${GOPAY_EN_CLIENTID}
["GOPAY_EN_CLIENTSECRET"]=${GOPAY_EN_CLIENTSECRET}
["GOPAY_CS_GOID"]=${GOPAY_CS_GOID}
["GOPAY_CS_CLIENTID"]=${GOPAY_CS_CLIENTID}
["GOPAY_CS_CLIENTSECRET"]=${GOPAY_CS_CLIENTSECRET}

["AKENEO_ENABLED"]=${AKENEO_ENABLED}
["AKENEO_BASE_URI"]=${AKENEO_BASE_URI}
["AKENEO_CLIENT_ID"]=${AKENEO_CLIENT_ID}
["AKENEO_SECRET"]=${AKENEO_SECRET}
["AKENEO_USER"]=${AKENEO_USER}
["AKENEO_PASSWORD"]=${AKENEO_PASSWORD}

["SSFWCC_BRIDGE_ENABLED"]=${SSFWCC_BRIDGE_ENABLED}
["SSFWCC_BRIDGE_BASE_URI"]=${SSFWCC_BRIDGE_BASE_URI}
["SSFWCC_BRIDGE_USER"]=${SSFWCC_BRIDGE_USER}
["SSFWCC_BRIDGE_PASSWORD"]=${SSFWCC_BRIDGE_PASSWORD}

["S3_ENDPOINT"]=${S3_ENDPOINT}
["S3_ACCESS_KEY"]=${S3_ACCESS_KEY}
["S3_SECRET"]=${S3_SECRET}
["S3_BUCKET_NAME"]=${PROJECT_NAME}

["CDN_DOMAIN"]=${CDN_DOMAIN}

["PACKETERY_ENABLED"]=${PACKETERY_ENABLED}
["PACKETERY_REST_API_URL"]=${PACKETERY_REST_API_URL}
["PACKETERY_API_PASSWORD"]=${PACKETERY_API_PASSWORD}
["PACKETERY_SENDER"]=${PACKETERY_SENDER}

["SENTRY_DSN"]=${SENTRY_DSN}
["SENTRY_ENVIRONMENT"]=${CI_ENVIRONMENT_SLUG}
["SENTRY_RELEASE"]=${CI_COMMIT_SHORT_SHA}
)

declare -A STOREFRONT_ENVIRONMENT_VARIABLES=(
["GTM_ID"]=${GTM_ID}
["SENTRY_DSN"]=${SENTRY_DSN}
["SENTRY_ENVIRONMENT"]=${CI_ENVIRONMENT_SLUG}
["PACKETERY_API_KEY"]=${PACKETERY_API_KEY}
["LIGHTGALLERY_LICENCE_KEY"]=${LIGHTGALLERY_LICENCE_KEY}
["REDIS_PREFIX"]=${PROJECT_NAME}

["CDN_DOMAIN"]=${CDN_DOMAIN}
)

declare -A CRON_INSTANCES=(
["cron"]='*/5 * * * *'
["cron-service"]='*/5 * * * *'
["cron-export"]='*/5 * * * *'
["cron-products"]='*/5 * * * *'
)

VARS=(
TAG
STOREFRONT_TAG
PROJECT_NAME
BASE_PATH
CI_ENVIRONMENT_SLUG
SENTRY_DSN
)

source "${DEPLOY_TARGET_PATH}/functions.sh"
source "${DEPLOY_TARGET_PATH}/parts/domains.sh"
source "${BASE_PATH}/deploy/parts/whitelist-ip.sh"
source "${DEPLOY_TARGET_PATH}/parts/environment-variables.sh"
source "${DEPLOY_TARGET_PATH}/parts/kubernetes-variables.sh"
source "${DEPLOY_TARGET_PATH}/parts/cron.sh"
source "${DEPLOY_TARGET_PATH}/parts/autoscaling.sh"
source "${DEPLOY_TARGET_PATH}/parts/deploy.sh"
}

function merge() {
source "${BASE_PATH}/vendor/shopsys/deployment/deploy/functions.sh"
merge_configuration
}

case "$1" in
"deploy")
deploy
;;
"merge")
merge
;;
*)
echo "invalid option"
exit 1
;;
esac
29 changes: 29 additions & 0 deletions app/deploy/parts/whitelist-ip.sh
@@ -0,0 +1,29 @@
echo -n "Whitelist IP addresses "

assertVariable "RUNNING_PRODUCTION"
assertVariable "CONFIGURATION_TARGET_PATH"
assertVariable "DOMAINS"
assertVariable "WHITELIST_IP"

# Do not run this script if there is no domains with HTTP AUTH
if [ ${RUNNING_PRODUCTION} -eq "1" ] && [ ${#FORCE_HTTP_AUTH_IN_PRODUCTION[@]} -ne "1" ]; then
echo -e "[${YELLOW}SKIP${NO_COLOR}]"
return
fi

DOMAIN_ITERATOR=0

# Configure IP addresses for Domain with HTTP auth
for DOMAIN in ${DOMAINS[@]}; do
INGRESS_FILENAME="ingress-${DOMAIN_ITERATOR}.yaml"

if [ ${RUNNING_PRODUCTION} -eq "0" ] || containsElement ${DOMAIN} ${FORCE_HTTP_AUTH_IN_PRODUCTION[@]}; then
yq write --inplace "${CONFIGURATION_TARGET_PATH}/ingress/${INGRESS_FILENAME}" metadata.annotations."\"nginx.ingress.kubernetes.io/configuration-snippet\"" "satisfy any;
${WHITELIST_IP}
deny all;"
fi

DOMAIN_ITERATOR=$(expr $DOMAIN_ITERATOR + 1)
done

echo -e "[${GREEN}OK${NO_COLOR}]"
5 changes: 5 additions & 0 deletions app/docker/php-fpm/Dockerfile
Expand Up @@ -134,6 +134,11 @@ RUN composer install --optimize-autoloader --no-interaction --no-progress --no-d

RUN php phing build-deploy-part-1-db-independent clean


FROM production as production-project

RUN chmod +x ./deploy/deploy-project.sh && ./deploy/deploy-project.sh merge

########################################################################################################################

FROM base as ci
Expand Down
5 changes: 4 additions & 1 deletion gitlab/docker-compose-ci.yml
Expand Up @@ -50,13 +50,16 @@ services:
container_name: shopsys-framework-redis

selenium-server:
image: selenium/standalone-chrome:3.141.5
image: seleniarm/standalone-firefox:4.9.0
container_name: shopsys-framework-acceptance-tests
ports:
- "4400:4444"
- "5900:5900"
- "7900:7900"
environment:
- HUB_PORT_4444_TCP_ADDR=hub
- HUB_PORT_4444_TCP_PORT=4444
shm_size: '2GB'

elasticsearch:
build:
Expand Down
13 changes: 13 additions & 0 deletions gitlab/mirror-repository/gitlab-stage.yml
@@ -0,0 +1,13 @@
mirror-project-from-github:
stage: service
rules:
- if: '$MIRROR_ONLY == "true"'
when: always
needs: []
script:
- apk add git openssh-client
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- echo "$SSHKEY_MIRROR_PROJECT_BASE" | ssh-add -
- sh ./gitlab/mirror-repository/mirror.sh "shopsys-project-base" "git@github.com:shopsys/project-base.git" "$MIRROR_SOURCE_BRANCH" "git@gitlab.shopsys.cz:ss6-projects/project-base.git" "$MIRROR_TARGET_BRANCH"
20 changes: 20 additions & 0 deletions gitlab/mirror-repository/mirror.sh
@@ -0,0 +1,20 @@
#!/bin/sh
set -euxo pipefail

WORKING_DIR_NAME=$1

SOURCE_URL=$2
SOURCE_BRANCH=$3

TARGET_URL=$4
TARGET_BRANCH=$5

if [[ -d "$WORKING_DIR_NAME" ]]; then
rm -rf "$WORKING_DIR_NAME"
fi

git clone --bare --single-branch --branch="$SOURCE_BRANCH" "$SOURCE_URL" "$WORKING_DIR_NAME"
cd "$WORKING_DIR_NAME"


git push --force "$TARGET_URL" "$SOURCE_BRANCH":"$TARGET_BRANCH"
2 changes: 1 addition & 1 deletion scripts/install.sh
Expand Up @@ -7,7 +7,7 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
echo "Installing application inside a php-fpm container"

docker-compose exec -T php-fpm composer install
docker-compose exec -T php-fpm ./phing db-create test-db-create frontend-api-enable build-demo-dev-quick error-pages-generate
docker-compose exec -T php-fpm ./phing db-create test-db-create frontend-api-generate-new-keys build-demo-dev-quick error-pages-generate

echo "Your application is now ready under http://127.0.0.1:8000 and second domain under http://127.0.0.2:8000"
echo "Administration is ready under http://127.0.0.1:8000/admin, you can log in using username 'admin' and password 'admin123'"

0 comments on commit e2853e7

Please sign in to comment.