#142 Renew devops configs (#195)

- Skip problem selenium test - Review#2 fix. Annotate commented string - Review#2 fix. Rm git install from dockerfile.dev - Fix forgotten env vars. Rm pdd task in favor of se#345 - Resurrect#1 gulp build - Launch tests #1 - Fix forgotten drone file - Review#1 fixes - Devops configs renew - Rm coala's redundant lint step
fidals · Jun 15, 2018 · 025bae7 · 025bae7 · 0pdd · Jun 15, 2018
1 parent d88d4bf
commit 025bae7
Show file tree

Hide file tree

Showing 26 changed files with 406 additions and 272 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -12,7 +12,7 @@ pipeline:
       branch: master
 
   npm:
-    image: node
+    image: node:slim
     environment:
       - DEPS_DIR=/usr/app/deps
     commands:
@@ -31,38 +31,36 @@ pipeline:
     environment:
       - TEST_ENV=true
       - DJANGO_SETTINGS_MODULE=stroyprombeton.settings.dev
-      - DATABASE_URL=postgres://postgres:test@postgres/test
-      - BROCKER_URL=amqp://rabbitmq:test@rabbitmq:5672/
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=test
+      - POSTGRES_URL=postgres
+      - POSTGRES_DB=test
+      - RABBITMQ_DEFAULT_USER=rabbitmq
+      - RABBITMQ_DEFAULT_PASS=test
+      - RABBITMQ_URL=rabbitmq
+      - RABBITMQ_PORT=5672
       - SELENIUM_URL=http://selenium:4444/wd/hub
     commands:
+      - python manage.py migrate
       - python manage.py collectstatic --noinput
-      - python manage.py test -v 3 --liveserver=test:8020-8030 --parallel=6
+      - python manage.py test --liveserver=test:8021-8029 --parallel
     volumes:
       - /tmp/cache/drone/stroyprombeton/site-packages/${DRONE_COMMIT_SHA}/site-packages:/usr/local/lib/python3.6/site-packages
     when:
       event: [push, pull_request]
       branch: master
 
-  pip-clean:
-    image: busybox
-    commands:
-      - rm -rf /site-packages/${DRONE_COMMIT_SHA}
-    volumes:
-      - /tmp/cache/drone/stroyprombeton/site-packages/:/site-packages/
-    when:
-      status: [ success, failure ]
-    when:
-      event: [push, pull_request]
-      branch: master
-
   lint:
     image: docker/compose:1.17.1
     commands:
       - cd docker
-      - docker-compose run --rm stb-lint
-      - docker-compose run --rm stb-pdd
+      - docker-compose run --rm lint
+      - docker-compose run --rm pdd
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      # @todo #142:60m Explore why drone `lint` and `docker-build` use .env
+      #  Both of them should not as it seems. Write doc why or rm them.
+      # in case if "Push Hooks" is enabled in Drone settings GUI
       - /root/prog/stroyprombeton/docker/.env:/drone/src/github.com/fidals/stroyprombeton/commit/${DRONE_COMMIT_SHA}/docker/.env
     when:
       event: [push, pull_request]
@@ -72,9 +70,7 @@ pipeline:
     image: docker/compose:1.17.1
     commands:
       - cd docker
-      - docker-compose -f docker-compose.yml build --no-cache stb-python
-      - docker-compose -f docker-compose-production.yml build --no-cache stb-source
-      - docker-compose -f docker-compose-production.yml build --no-cache stb-python
+      - docker-compose -f docker-compose-build.yml build --no-cache nodejs python-dev python-prod
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       # in case if "Push Hooks" is enabled in Drone settings GUI
@@ -92,7 +88,7 @@ pipeline:
       - DOCKER_HUB_PASSWORD=${DOCKER_HUB_PASSWORD}
     commands:
       - docker login -u $DOCKER_HUB_LOGIN -p $DOCKER_HUB_PASSWORD
-      - docker push fidals/stb-source:prod
+      - docker push fidals/stb-nodejs:prod
       - docker push fidals/stb:dev
       - docker push fidals/stb:prod
     volumes:
@@ -104,6 +100,24 @@ pipeline:
       event: push
       branch: master
 
+  pip-clean:
+    image: busybox
+    commands:
+      - rm -rf /site-packages/${DRONE_COMMIT_SHA}
+    volumes:
+      - /tmp/cache/drone/stroyprombeton/site-packages/:/site-packages/
+    when:
+      status: [ success, failure ]
+
+  npm-clean:
+    image: busybox
+    commands:
+      - rm -rf /drone/src/github.com/fidals/stroyprombeton/pull/${DRONE_PULL_REQUEST}/node_modules/*
+    volumes:
+      - /tmp/cache/drone/stroyprombeton/node_modules:/drone/src/github.com/fidals/stroyprombeton/pull/${DRONE_PULL_REQUEST}/node_modules
+    when:
+      status: [ success, failure ]
+
 services:
   postgres:
     group: build
@@ -122,10 +136,11 @@ services:
 
   selenium:
     group: build
-    image: selenium/standalone-chrome:3.6.0
+    image: selenium/standalone-chrome-debug:3.10.0
     environment: # https://github.com/SeleniumHQ/docker-selenium/issues/392
       - DBUS_SESSION_BUS_ADDRESS=/dev/null
+      - SCREEN_WIDTH=1920
+      - SCREEN_HEIGHT=1080
     shm_size: 4G
     volumes: # https://github.com/SeleniumHQ/docker-selenium#running-the-images
       - /dev/shm:/dev/shm
-
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,8 @@ media/
 stroyprombeton/settings/local.py
 stb.json
 docker/.env
+docker/env_files/*
+!docker/env_files/*.dist
 celerybeat*
 fake_db.py
 .vscode/

diff --git a/docker/.env.dist b/docker/.env.dist
diff --git a/docker/Makefile b/docker/Makefile
@@ -1,61 +1,99 @@
 dc=docker-compose
 d=docker
-dcp=docker-compose -f docker-compose-production.yml
+dcb=$(dc) -f docker-compose-build.yml
+dcp=$(dc) -f docker-compose-production.yml
 
+.PHONY: migrate create-env build-static watch-static \
+build test backup restore \
+generate-production-static-data deploy
+
+
+# ---------------------- Dev section ----------------------
 migrate:
-	$(dc) exec stb-python python manage.py migrate
-
-# for local env
-static:
-	@docker-compose run --rm stb-nodejs bash -c "\
-		npm install && \
-		npm install -g gulp-cli && \
-		gulp build"
-
-# for local env
-build: static
-	$(dc) build stb-python
-
-# for local env
-test: static
-	$(dc) up -d stb-selenium
-	$(dc) exec stb-python python manage.py test --liveserver=stb-python:8001-8010 --parallel=8
+	$(dc) exec app python manage.py migrate
+
+fixtures:
+	$(dc) run --rm app bash -c "\
+		python manage.py loaddata stroyprombeton/fixtures/admin.json \
+		&& python manage.py loaddata stroyprombeton/fixtures/dump.json \
+	"
+
+prices:
+	$(dc) exec app python manage.py price
+
+create-env:
+	@bash ./create-env.sh
+
+create-config:
+	@bash ./create-config.sh
+
+build-static:
+	$(dc) run --rm nodejs gulp build
+
+watch-static:
+	$(dc) run --rm nodejs gulp watch
+
+collectstatic:
+	$(dc) exec app python manage.py collectstatic --noinput
+
+test: build-static
+	$(dc) up -d selenium
+	$(dc) exec app python manage.py test --liveserver=app:8001-8009 --parallel
 	$(dc) stop
 
-# for local env
 lint:
-	$(dc) up -d stb-python
-	$(d) run --rm --volumes-from stb-python --workdir=/usr/app/src -it coala/base coala
+	$(dc) up -d app
+	$(dc) run --rm lint
 	$(dc) stop
 
-# for local env
-restore:
-	@bash ../etc/stb-backup-restore.sh
 
-# for serv env
+deploy-dev:
+	$(MAKE) create-env
+	# uncomment in #167
+	# $(MAKE) create-config
+	$(dc) pull
+	$(dc) up -d app
+	$(MAKE) build-static
+	$(MAKE) migrate
+	# Create admin user with login/pass: admin/asdfjkl;
+	$(MAKE) fixtures
+	# Launch "collectstatic" not in static recipe because ManifestStaticStorage writes to db
+	$(MAKE) collectstatic
+	# to make fresh collected static visible immediately
+	$(dc) stop app && $(dc) up -d app
+
+
+# ---------------------- Production deploy section ----------------------
+# @todo #142:60m Resurrect backup system.
+#  Right now it's too dangerous and not used.
 backup:
-	$(dcp) up stb-backup-data
-
-# start dev environment
-dev: build
-	$(dc) up -d stb-python
-	$(dc) exec stb-python python manage.py migrate
-	# launch "collectstatic" not in static recipe because ManifestStaticStorage writes to db
-	$(dc) exec stb-python python manage.py collectstatic --noinput
-	$(dc) stop && $(dc) up -d  # to make fresh collected static visible immediately
-
-# for serv env
-build-production: build
-	$(dcp) build
-	$(dcp) push
-
-deploy: backup
+	$(dcp) run --rm backup-data sh /usr/bin/entrypoint.sh
+
+restore:
+	@bash ../etc/backup/backup-restore.sh
+
+generate-production-static-data:
+	$(dcp) exec app python manage.py price
+
+# drone should do this in working flow.
+# But if drone don't do this for some reasons,
+# you can run this recipe from local machine.
+prepare-deploy:
+	rm -rf front/build
+	$(dc) stop app && $(dcp) stop app
+	$(dc) rm -f app && $(dcp) rm -f app
+	$(MAKE) build-static
+	$(dcb) build --no-cache python-dev python-prod
+	$(dcb) push python-dev python-prod
+
+deploy:
 	$(dcp) pull
 	$(dcp) stop
-	$(dcp) rm -f stb-source
-	$(dcp) up -d
-	$(dcp) exec stb-python python manage.py migrate
-	# launch "collectstatic" not in build env, but on deploy.
-	# Because ManifestStaticStorage writes to db
-	$(dcp) exec stb-python python manage.py collectstatic --noinput
-	$(dcp) stop && $(dcp) up -d  # to make fresh collected static visible immediately
+	# to flush nginx's inner static caches
+	$(dcp) rm -f app nginx && $(dcp) up -d app nginx
+	$(dcp) exec app python manage.py migrate
+	$(MAKE) -j generate-production-static-data
+	# Launch "collectstatic" not in static recipe because ManifestStaticStorage writes to db
+	$(dcp) exec app python manage.py collectstatic --noinput
+	# to make fresh collected static visible immediately
+	$(dcp) stop && $(dcp) up -d
diff --git a/docker/README.md b/docker/README.md