Merge 129f63c into befe709

README.md

@@ -39,23 +39,26 @@ Based on the documentation, [Running Airflow in Docker](https://airflow.apache.o
 ### Tasks
 1. List all the airflow tasks using `cap -AT airflow`
 ```
-cap airflow:build      # run docker-compose build for airflow
-cap airflow:init       # run docker-compose init for airflow
-cap airflow:ps         # show running docker processes
-cap airflow:restart    # restart airflow
-cap airflow:start      # start airflow
-cap airflow:stop       # stop and remove all running docker containers
-cap airflow:webserver  # restart webserver
+cap airflow:build          # run docker-compose build for airflow
+cap airflow:init           # run docker-compose init for airflow
+cap airflow:ps             # show running docker processes
+cap airflow:restart        # restart airflow
+cap airflow:start          # start airflow
+cap airflow:stop           # stop and remove all running docker containers
+cap airflow:stop_release   # stop old release and remove all old running docker containers
+cap airflow:webserver      # restart webserver
 ```
 
 ### Do the first time you bring up Libsys-Airflow:
-1. `cap {stage} airflow:deploy`
+1. `cap ${stage} deploy deploy:start`
 1. Follow the instructions for [shared_configs/libsys-airflow](https://github.com/sul-dlss/shared_configs/tree/libsys-airflow#readme)
-1. `cap {stage} airflow:build`
-1. `cap {stage} airflow:init`
-1. `cap {stage} airflow:start`
 1. Visit https://sul-libsys-airflow-{stage}.stanford.edu and complete the remaining steps.
 
+## For subsequent deploys
+`cap ${stage} deploy deploy:restart`
+
+This will stop and remove the docker images for the previous release and start up a new one.
+
 ### For Aeon and Lobbytrack API calls
 1. In the Airflow UI under Admin > Variables, import the `aeon-variables.json` and the `lobbytrack-variables.json` files from [shared_configs](https://github.com/sul-dlss/shared_configs).
 

config/deploy.rb

@@ -20,7 +20,7 @@
 set :linked_dirs, %w[config]
 
 # Default value for keep_releases is 5
-set :keep_releases, 3
+set :keep_releases, 2
 
 before 'deploy:cleanup', 'fix_permissions'
 
@@ -33,19 +33,39 @@
   end
 end
 
-task :deploy do
-  on roles(:app) do
-    invoke 'airflow:stop'
-    execute "cd #{release_path} && source #{fetch(:venv)} && pip3 install -r requirements.txt"
-    execute "cp #{release_path}/config/.env #{release_path}/."
-    execute "cd #{release_path} && git clone #{fetch(:migration)} migration"
-    execute "chmod +x #{release_path}/migration/create_folder_structure.sh"
-    execute "cd #{release_path}/migration && ./create_folder_structure.sh"
-    invoke 'airflow:start'
+namespace :deploy do
+  desc 'deploy airflow when an instance is not currently running'
+  task :start do
+    on roles(:app) do
+      invoke 'airflow:install'
+      invoke 'airflow:build'
+      invoke 'airflow:init'
+      invoke 'airflow:start'
+    end
+  end
+
+  desc 'deploy airflow when an instance is currently running'
+  task :restart do
+    on roles(:app) do
+      invoke 'airflow:stop_release'
+      invoke 'airflow:install'
+      invoke 'airflow:start'
+    end
   end
 end
 
 namespace :airflow do
+  desc 'install airflow dependencies'
+  task :install do
+    on roles(:app) do
+      execute "cd #{release_path} && source #{fetch(:venv)} && pip3 install -r requirements.txt"
+      execute "cp #{release_path}/config/.env #{release_path}/."
+      execute "cd #{release_path} && git clone #{fetch(:migration)} migration"
+      execute "chmod +x #{release_path}/migration/create_folder_structure.sh"
+      execute "cd #{release_path}/migration && ./create_folder_structure.sh"
+    end
+  end
+
   desc 'show running docker processes'
   task :ps do
     on roles(:app) do
@@ -95,4 +115,13 @@
       invoke 'airflow:start'
     end
   end
+
+  desc 'stop old release and remove all old running docker containers'
+  task :stop_release do
+    on roles(:app) do
+      execute "docker image prune -f"
+      execute "cd #{release_path} && releases=($(ls -tr ../.)) && cd ../${releases[0]} && source #{fetch(:venv)} && docker-compose stop"
+      execute "docker rm $(docker ps --filter status=exited -q)"
+    end
+  end
 end