Merge pull request #898 from pulibrary/i861-move-to-lando

Switch to using lando for dev and test
pulibrary · May 5, 2021 · d23c8d3 · d23c8d3
2 parents a90574d + c9ddc92
commit d23c8d3
Show file tree

Hide file tree

Showing 12 changed files with 98 additions and 96 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -81,11 +81,14 @@ jobs:
           POMEGRANATE_DB_HOST: localhost
           POMEGRANATE_DB_USERNAME: pomegranate
           POMEGRANATE_DB_PASSWORD: ""
+          SOLR_URL: "http://localhost:8984/solr/blacklight-core-test"
       - image: postgres:10.3
         environment:
           POSTGRES_DB: dpul_test
           POSTGRES_USER: pomegranate
           POSTGRES_PASSWORD: ""
+      - image: solr:7.7-alpine
+        command: bin/solr -cloud -noprompt -f -p 8984
     steps:
       - attach_workspace:
           at: '~/pomegranate'
@@ -109,9 +112,12 @@ jobs:
       # Bundle install dependencies
       - run: bundle install --path vendor/bundle
       - run:
-          command: bundle exec rake dpul:test
-          background: true
-      - run: bin/jetty_wait
+          name: Load config into solr
+          command: |
+            cd ~/pomegranate/solr/config
+            zip -1 -r solr_config.zip ./*
+            curl -H "Content-type:application/octet-stream" --data-binary @solr_config.zip "http://localhost:8984/solr/admin/configs?action=UPLOAD&name=dpul"
+            curl -H 'Content-type: application/json' http://localhost:8984/api/collections/ -d '{create: {name: blacklight-core-test, config: dpul, numShards: 1}}'
       - run: bundle exec rake db:create
       - run: bundle exec rake db:migrate
       - run: bundle exec rspec spec

diff --git a/.lando.yml b/.lando.yml
@@ -0,0 +1,27 @@
+name: dpul
+services:
+  dpul_development_db:
+    type: postgres:10
+    portforward: true
+    creds:
+      database: dpul_development
+  dpul_test_db:
+    type: postgres:10
+    portforward: true
+    creds:
+      database: dpul_test
+  dpul_development_solr:
+    type: solr:7
+    portforward: true
+    core: blacklight-core
+    config:
+      dir: solr/config
+  dpul_test_solr:
+    type: solr:7
+    portforward: true
+    core: blacklight-core-test
+    config:
+      dir: solr/config
+  dpul_redis:
+    type: redis:6
+    portforward: true
diff --git a/.solr_wrapper b/.solr_wrapper
diff --git a/Gemfile b/Gemfile
@@ -38,7 +38,6 @@ group :test do
 end
 
 group :development do
-  gem 'foreman'
   gem 'puma'
   gem 'web-console', '~> 2.0'
 end
@@ -70,7 +69,6 @@ gem 'rsolr', '~> 1.0.6'
 gem 'sidekiq', '< 6'
 gem 'sitemap_generator'
 gem 'sneakers'
-gem 'solr_wrapper'
 gem 'sprockets', '~> 3.7'
 gem 'sprockets-es6'
 gem 'sprockets-rails'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -234,8 +234,6 @@ GEM
     faraday_middleware (0.14.0)
       faraday (>= 0.7.4, < 1.0)
     ffi (1.14.2)
-    foreman (0.85.0)
-      thor (~> 0.19.1)
     friendly_id (5.4.0)
       activerecord (>= 4.0.0)
     github-markup (3.0.4)
@@ -417,7 +415,6 @@ GEM
     responders (3.0.0)
       actionpack (>= 5.0)
       railties (>= 5.0)
-    retriable (3.1.2)
     riiif (2.1.0)
       deprecation (>= 1.0.0)
       iiif-image-api (~> 0.1.0)
@@ -516,11 +513,6 @@ GEM
       rake (~> 12.3)
       serverengine (~> 2.1.0)
       thor
-    solr_wrapper (2.1.0)
-      faraday
-      retriable
-      ruby-progressbar
-      rubyzip
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -621,7 +613,6 @@ DEPENDENCIES
   devise-guests (~> 0.3)
   devise_invitable
   factory_bot_rails
-  foreman
   graphql-client
   honeybadger (~> 2.0)
   iiif-presentation
@@ -655,7 +646,6 @@ DEPENDENCIES
   simplecov
   sitemap_generator
   sneakers
-  solr_wrapper
   sprockets (~> 3.7)
   sprockets-es6
   sprockets-rails
@@ -670,4 +660,4 @@ DEPENDENCIES
   webpacker (>= 4.0.x)
 
 BUNDLED WITH
-   2.2.6
+   2.2.16
diff --git a/README.md b/README.md
@@ -26,27 +26,28 @@ bundle exec rake db:create
 bundle exec rake db:migrate
 ```
 
-After setup, run DPUL locally either with
+After setup, start solr, postgresql and redis with:
 
-`bundle exec foreman start`
+`bundle exec rake servers:start`
 
-to run everything at once, or, in separate terminal windows run each command
-listed in `Procfile`
+Then, start webpack-dev-server, sidekiq and rails, each in their own terminal window:
+`bin/webpack-dev-server`
+`bundle exec sidekiq`
+`bin/rails s -p 3000`
 
 ### Importing Data:
 
 1. Log in once via CAS
 2. Run `rake dpul:site_admin`
 3. Click "Create a New Collection"
 4. Select a small collection and hit "Save"
-  - To find a small collection: go to Figgy, submit a blank search, open the facet collection and click 'more', and page to the low-count collections 
+  - To find a small collection: go to Figgy, submit a blank search, open the facet collection and click 'more', and page to the low-count collections
 5. Either wait for a solr commit or manually commit in the rails console with
    `Blacklight.default_index.connection.commit`
 
 ### Running Tests
 
 ```sh
-bundle exec rake dpul:test
 bundle exec rspec
 ```
 

diff --git a/Rakefile b/Rakefile
@@ -6,8 +6,6 @@
 require File.expand_path('config/application', __dir__)
 require 'rubocop/rake_task'
 require 'sneakers/tasks'
-require 'solr_wrapper'
-require 'solr_wrapper/rake_task'
 
 Rails.application.load_tasks
 

diff --git a/config/application.rb b/config/application.rb
@@ -6,6 +6,7 @@
 require 'open-uri'
 require 'susy'
 require 'breakpoint'
+require_relative 'lando_env'
 
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.

diff --git a/config/blacklight.yml b/config/blacklight.yml
@@ -2,22 +2,22 @@
 # each environment can have a jetty_path with absolute or relative
 # (to app root) path to a jetty/solr install. This is used
 # by the rake tasks that start up solr automatically for testing
-# and by rake solr:marc:index.  
+# and by rake solr:marc:index.
 #
 # jetty_path is not used by a running Blacklight application
 # at all. In general you do NOT need to deploy solr in Jetty, you can deploy it
-# however you want.  
+# however you want.
 # jetty_path is only required for rake tasks that need to know
-# how to start up solr, generally for automated testing. 
+# how to start up solr, generally for automated testing.
 
 development:
   adapter: solr
-  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr/blacklight-core" %>
+  url: <%= ENV['SOLR_URL'] || "http://#{ENV['lando_dpul_development_solr_conn_host']}:#{ENV['lando_dpul_development_solr_conn_port']}/solr/blacklight-core" %>
 test:
   adapter: "solr"
-  url: <%= "http://127.0.0.1:#{ENV['TEST_JETTY_PORT'] || 8984}/solr/blacklight-core-test" %>
+  url: <%= ENV['SOLR_URL'] || "http://#{ENV['lando_dpul_test_solr_conn_host']}:#{ENV['lando_dpul_test_solr_conn_port']}/solr/blacklight-core-test" %>
 production: &production
   adapter: solr
   url: <%= ENV['POMEGRANATE_SOLR_URL'] || "http://127.0.0.1:8980/solr/blacklight-core" %>
-staging: 
+staging:
   <<: *production
diff --git a/config/database.yml b/config/database.yml
@@ -1,18 +1,22 @@
-development: &default
+default: &default
   adapter: postgresql
+  encoding: unicode
+
+development: &development
+  <<: *default
+  host: <%= ENV["lando_dpul_development_db_conn_host"] || ENV["POMEGRANATE_DB_HOST"] || "localhost" %>
+  port: <%= ENV["lando_dpul_development_db_conn_port"] || "5432" %>
+  username: <%= ENV["lando_dpul_development_db_creds_user"] || ENV["POMEGRANATE_DB_USERNAME"] %>
+  password: <%= ENV["lando_dpul_development_db_creds_password"] || ENV["POMEGRANATE_DB_PASSWORD"] %>
   database: dpul_development
-  encoding: utf8
-  min_messages: warning
-  pool: <%= Integer(ENV.fetch("DB_POOL", 30)) %>
-  reaping_frequency: <%= Integer(ENV.fetch("DB_REAPING_FREQUENCY", 10)) %>
-  timeout: 5000
 
 test:
   <<: *default
-  host: <%= ENV["POMEGRANATE_DB_HOST"] %>
+  host: <%= ENV["lando_dpul_test_db_conn_host"] || ENV["POMEGRANATE_DB_HOST"] || "localhost" %>
+  port: <%= ENV["lando_dpul_test_db_conn_port"] || "5432" %>
+  username: <%= ENV["lando_dpul_test_db_creds_user"] || ENV["POMEGRANATE_DB_USERNAME"] %>
+  password: <%= ENV["lando_dpul_test_db_creds_password"] || ENV["POMEGRANATE_DB_PASSWORD"] %>
   database: dpul_test
-  username: <%= ENV["POMEGRANATE_DB_USERNAME"] %>
-  password: <%= ENV["POMEGRANATE_DB_PASSWORD"] %>
 
 production: &deploy
   <<: *default

diff --git a/config/lando_env.rb b/config/lando_env.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+if Rails.env.development? || Rails.env.test?
+  begin
+    lando_services = JSON.parse(`lando info --format json`, symbolize_names: true)
+    lando_services.each do |service|
+      service[:external_connection]&.each do |key, value|
+        ENV["lando_#{service[:service]}_conn_#{key}"] = value
+      end
+      next unless service[:creds]
+
+      service[:creds].each do |key, value|
+        ENV["lando_#{service[:service]}_creds_#{key}"] = value
+      end
+    end
+  rescue StandardError
+    nil
+  end
+end
diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake
@@ -13,55 +13,28 @@ end
 if Rails.env.development? || Rails.env.test?
   require "factory_bot"
 
+  namespace :servers do
+    desc "Start solr and postgres servers using lando."
+    task :start do
+      system("lando start")
+      system("rake db:create")
+      system("rake db:migrate")
+      system("rake db:migrate RAILS_ENV=test")
+    end
+
+    desc "Stop lando solr and postgres servers."
+    task :stop do
+      system("lando stop")
+    end
+  end
+
   namespace :dpul do
     desc 'Make first user a site admin'
     task site_admin: :environment do
       user = User.first
       user.roles.create role: 'admin', resource: Spotlight::Site.instance
       puts "Made #{user} a site admin!"
     end
-
-    desc "Start solr server for testing."
-    task :test do
-      shared_solr_opts = { managed: true, verbose: true, persist: false, download_dir: 'tmp' }
-      shared_solr_opts[:version] = ENV['SOLR_VERSION'] if ENV['SOLR_VERSION']
-
-      SolrWrapper.wrap(shared_solr_opts.merge(port: 8984, instance_dir: 'tmp/blacklight-core-test')) do |solr|
-        solr.with_collection(name: "blacklight-core-test", dir: Rails.root.join("solr", "config").to_s) do
-          puts "Solr running at http://localhost:8984/solr/blacklight-core-test/, ^C to exit"
-          begin
-            sleep
-          rescue Interrupt
-            puts "\nShutting down..."
-          end
-        end
-      end
-    end
-
-    desc "Start solr server for development."
-    task :development do
-      SolrWrapper.wrap(managed: true, verbose: true, port: 8983, instance_dir: 'tmp/blacklight-core', persist: true, download_dir: 'tmp') do |solr|
-        solr.with_collection(name: "blacklight-core", dir: Rails.root.join("solr", "config").to_s) do
-          puts "Setup solr"
-          puts "Solr running at http://localhost:8983/solr/blacklight-core/, ^C to exit"
-          begin
-            if ENV['ENABLE_RAILS']
-              # If HOST specified, bind to that IP with -b
-              server_options = " -b #{ENV['HOST']}" if ENV['HOST']
-              IO.popen("rails server#{server_options}") do |io|
-                io.each do |line|
-                  puts line
-                end
-              end
-            else
-              sleep
-            end
-          rescue Interrupt
-            puts "\nShutting down..."
-          end
-        end
-      end
-    end
   end
 
   namespace :clean do
@@ -84,14 +57,6 @@ if Rails.env.development? || Rails.env.test?
       end
     end
 
-    namespace :test do
-      desc "Cleanup test servers"
-      task :solr do
-        SolrWrapper.instance(managed: true, verbose: true, port: 8984, instance_dir: 'tmp/blacklight-core-test', persist: false).remove_instance_dir!
-        puts "Cleaned up test solr server."
-      end
-    end
-
     namespace :development do
       desc "Delete development database and index data"
       task all: :environment do