Split external assets rake task into two (#657)

* Split external assets rake task into two

* make CI import liquid templates from ext-assets for smoke tests

* attempt to fix bizarre yaml parse error

* make test.yml use two space indentation like normal yml

* prioritize champaign translations over asset trans in test env

circle.yml

@@ -15,7 +15,7 @@ dependencies:
     # build the image
     - npm install
     - bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 --without development test
-    - RAILS_ENV=production bundle exec rake deploy:precompile_assets[$CUSTOM_ASSETS_URL,$CUSTOM_ASSETS_CREDENTIALS,$CIRCLE_BRANCH]
+    - RAILS_ENV=production bundle exec rake assets:download_and_precompile[$CUSTOM_ASSETS_URL,$CUSTOM_ASSETS_CREDENTIALS,$CIRCLE_BRANCH,$EXTERNAL_ASSET_PATHS]
     - docker build -t soutech/champaign_web:$CIRCLE_SHA1 .
 
 test:

circle/test.template.env

@@ -8,4 +8,7 @@ BRAINTREE_PRIVATE_KEY=324
 CIRCLECI=true
 CI=true
 COVERALLS_REPO_TOKEN=$COVERALLS_REPO_TOKEN
+EXTERNAL_ASSET_PATHS=$EXTERNAL_ASSET_PATHS
+EXTERNAL_LIQUID_PATH=$EXTERNAL_LIQUID_PATH
+EXTERNAL_TRANSLATION_PATH=$EXTERNAL_TRANSLATION_PATH
 GOCARDLESS_TOKEN='halpiamatoken'

config/application.rb

@@ -39,17 +39,6 @@ class Application < Rails::Application
 
     config.active_record.observers = :liquid_partial_observer
 
-    if Settings.external_asset_paths.present?
-      path_list = Settings.external_asset_paths.try(:split, ":")
-      config.assets.paths += path_list
-
-      if Settings.external_translation_path.present?
-        path_list.each do |directory|
-          config.i18n.load_path += Dir[File.join(directory, Settings.external_translation_path, '*.{rb,yml}')]
-        end
-      end
-    end
-
     # We're using Redis as our cache. Configure that here.
     # we use 'redis' as the host name because that's configured by docker
     # during our setup as the host where our redis instance is stored.

config/initializers/locale.rb

@@ -0,0 +1,13 @@
+# include the locale files from external asset directories, if relevant
+if Settings.external_asset_paths.present? && Settings.external_translation_path.present?
+  Settings.external_asset_paths.try(:split, ":").each do |directory|
+    Rails.application.config.i18n.load_path += Dir[File.join(directory, Settings.external_translation_path, '*.{rb,yml}')]
+  end
+  # in the test environement, we want the translations from the Champaign repo to override those in
+  # the asset repo because some of the integration specs involve VCR casettes including language
+  # from the Champaign translations
+  if Rails.env.test?
+    champaign_locales = Dir[Rails.root.join('config', 'locales', '*.{rb,yml}')]
+    Rails.application.config.i18n.load_path += champaign_locales
+  end
+end

config/settings/production.yml

@@ -94,10 +94,15 @@ gocardless:
   gbp_charge_day: 20
   secret: <%= ENV['GOCARDLESS_SECRET'] %>
 
+home_page_url: <%= ENV['HOME_PAGE_URL'] %>
+
 # Colon separated list of paths that will be added to config.assets.path
 # http://guides.rubyonrails.org/v4.2/asset_pipeline.html#asset-organization
-external_assets_paths: <%= ENV['EXTERNAL_ASSETS_PATHS'] %>
-home_page_url: <%= ENV['HOME_PAGE_URL'] %>
+external_asset_paths: '<%= ENV["EXTERNAL_ASSET_PATHS"] %>'
+external_css_path: '<%= ENV["EXTERNAL_CSS_PATH"] %>'
+external_js_path: '<%= ENV["EXTERNAL_JS_PATH"] %>'
+external_translation_path: '<%= ENV["EXTERNAL_TRANSLATION_PATH"] %>'
+external_liquid_path: '<%= ENV["EXTERNAL_LIQUID_PATH"] %>'
 default_asset_branch: <%= ENV['DEFAULT_ASSET_BRANCH'] %>
 
 

config/settings/test.yml

@@ -12,38 +12,42 @@
 # config/settings/#{environment}.yml will override the settings in this file, and settings in
 # config/settings/#{environment}.local.yml will override the settings in that file.
 
+external_asset_paths: '<%= ENV["EXTERNAL_ASSET_PATHS"] %>'
+external_translation_path: '<%= ENV["EXTERNAL_TRANSLATION_PATH"] %>'
+external_liquid_path: '<%= ENV["EXTERNAL_LIQUID_PATH"] %>'
+
 # General Champaign settings.
-     secret_key_base: 'a fake secret key'
-     omniauth_client_secret: 'a fake client secret'
-     omniauth_client_id: 'a fake client_id'
-     homepage_url: 'http://sumofus.org'
-     facebook_url: 'https://www.facebook.com/SumOfUs-181924628560212/'
-     oauth_domain_whitelist:
-       - 'example.com'
+secret_key_base: 'a fake secret key'
+omniauth_client_secret: 'a fake client secret'
+omniauth_client_id: 'a fake client_id'
+homepage_url: 'http://sumofus.org'
+facebook_url: 'https://www.facebook.com/SumOfUs-181924628560212/'
+oauth_domain_whitelist:
+  - 'example.com'
 
 # AWS Config Variables. These Variables will cause Champaign to attempt to send background processing information, like
 # page creation/update information or page signature events to a background processor listening on the specified
 # queue. Only set these settings to values if you want Champaign to take these actions. Disabled by default, except
 # in production situations.
-     aws_access_key_id: 'fakeaccesskeyid'
-     aws_secret_access_key: 'fakesecretaccesskey'
-     aws_region: 'US-West-2'
-     sqs_queue_url: 'http://example.com'
-     ak_processor_url: 'http://example.com'
+aws_access_key_id: 'fakeaccesskeyid'
+aws_secret_access_key: 'fakesecretaccesskey'
+aws_region: 'US-West-2'
+sqs_queue_url: 'http://example.com'
+ak_processor_url: 'http://example.com'
 
 
 # Direct ActionKit Connection variables.
-     ak_api_url: 'https://act.sumofus.org/rest/v1'
-     ak_username: 'ak_username'
-     ak_password: 'ak_password'
+ak_api_url: 'https://act.sumofus.org/rest/v1'
+ak_username: 'ak_username'
+ak_password: 'ak_password'
 
 
 # CloudFront config Variables. These variables are used to configure static asset serving via cloudfront, a CDN. Disabled
 # by default in development, they should be overridden in production settings to speed up Champaign's page performance.
-     cloudfront_url: 'http://fake-cloudfront.com'
-     s3_asset_bucket: 'fake_s3_asset_bucket'
-     rails_serve_static_assets: false
-     compile_static: false
+cloudfront_url: 'http://fake-cloudfront.com'
+s3_asset_bucket: 'fake_s3_asset_bucket'
+rails_serve_static_assets: false
+compile_static: false
 
 
 # NewRelic Connection information. This variable, if set to a valid value, will cause the NewRelic connector to send
@@ -52,54 +56,54 @@
 
 
 # Social media connection information. Useful for allowing connections to things like Facebook or Twitter.
-     facebook_app_id: 'fake_app_id'
-     share_progress_api_key: 'fake_api_key'
+facebook_app_id: 'fake_app_id'
+share_progress_api_key: 'fake_api_key'
 
 # Braintree payment processor
-     braintree:
-       merchant_id: "test_id"
-       public_key:  "test_public_key"
-       private_key:  "test_private_key"
-
-       merchants:
-         EUR: 'EUR'
-         GBP: 'GBP'
-         USD: 'USD'
-         AUD: 'AUD'
-         CAD: 'CAD'
-         NZD: 'NZD'
-
-       subscription_plans:
-         EUR: 'EUR'
-         GBP: 'GBP'
-         USD: '1'
-         AUD: 'AUD'
-         CAD: 'CAD'
-         NZD: 'NZD'
+braintree:
+  merchant_id: "test_id"
+  public_key:  "test_public_key"
+  private_key:  "test_private_key"
+
+  merchants:
+    EUR: 'EUR'
+    GBP: 'GBP'
+    USD: 'USD'
+    AUD: 'AUD'
+    CAD: 'CAD'
+    NZD: 'NZD'
+
+  subscription_plans:
+    EUR: 'EUR'
+    GBP: 'GBP'
+    USD: '1'
+    AUD: 'AUD'
+    CAD: 'CAD'
+    NZD: 'NZD'
 
 # GoCardless payment processor
-     gocardless:
-       token: <%= ENV["GOCARDLESS_TOKEN"] %>
-       environment: 'sandbox'
-       gbp_charge_day: 20
-       secret: 'monkey'
+gocardless:
+  token: <%= ENV["GOCARDLESS_TOKEN"] %>
+  environment: 'sandbox'
+  gbp_charge_day: 20
+  secret: 'monkey'
 
-     redis:
-       host: <%= ENV["REDIS_PORT_6379_TCP_ADDR"] %>
+redis:
+  host: <%= ENV["REDIS_PORT_6379_TCP_ADDR"] %>
 
-       port: <%= ENV["REDIS_PORT_6379_TCP_PORT"] %>
+  port: <%= ENV["REDIS_PORT_6379_TCP_PORT"] %>
 
-     robots: 'staging'
+robots: 'staging'
 
-     action_kit:
-       akid_secret: "secret_sauce"
+action_kit:
+  akid_secret: "secret_sauce"
 
-     host: https://example.com
+host: https://example.com
 
 # CircleCI test settings
-  # This will load the entire application. If code is not eager loaded constants are loaded on demand.
-  # Constant autoloading is not thread-safe, so if eager loading is not enabled multi-threading should not be allowed.
-  # This should take care of the Rack Mutex error on CircleCI.
-     eager_load_for_specs: true
+# This will load the entire application. If code is not eager loaded constants are loaded on demand.
+# Constant autoloading is not thread-safe, so if eager loading is not enabled multi-threading should not be allowed.
+# This should take care of the Rack Mutex error on CircleCI.
+eager_load_for_specs: true
 
-     home_page_url: 'http://example.com'
+home_page_url: 'http://example.com'

lib/tasks/deploy.rake → lib/tasks/assets.rake

@@ -1,30 +1,44 @@
-namespace :deploy do
-  # Downloads a tar from a URL, extracts its contents into ./tmp,
-  # and returns a colon separated list of paths belonging to the first level directories
-  # inside the original tar file.
+namespace :assets do
+
+  task :download_and_precompile, [:url_template, :credentials, :branch, :source_assets_path] => :environment do |t, args|
+    if args[:url_template].blank?
+      puts "Not including any external assets"
+      next
+    end
+
+    target_path = args[:source_assets_path] || "./tmp/assets_source"
+    FileUtils.mkdir_p target_path
+    Rake::Task['assets:download_external_assets'].invoke(target_path, args[:url_template], args[:credentials], args[:branch])
+    Rake::Task['assets:precompile_assets'].invoke(target_path)
+  end
+
+  # Downloads a tar from a URL, extracts its contents into <target_path>
   #
   # Params:
+  # * target_path: the dir path where the contents of the downloaded and extracted files will be put
   # * url: any http[s] url. Optionally it can include a tag <branch> that will be replaced with
-  #    the contents of the branch param.
+  #   the contents of the branch param.
   # * credentials: http basic auth credentials. The format should be user:password.
   # * branch: Mainly for Github URLs usage. The content of this param will replace the <branch> tag
-  #     in the url param.
+  #   in the url param.
   #
   # Example:
   #  rake deploy:precompile_assets["https://api.github.com/repos/organisation/repo/tarball/<branch>","deploy-user:secret","master"]
   desc "Download external assets"
-  task :precompile_assets, [:url, :credentials, :branch] => :environment do |t, args|
-    url_template = args[:url]
+  task :download_external_assets, [:target_path, :url_template, :credentials, :branch] => :environment do |t, args|
+    target_path = args[:target_path]
+    url_template = args[:url_template]
     credentials = args[:credentials]
     current_branch = args[:branch]
     tar_file_path = "./tmp/assets.tar"
 
 
-    if url_template.blank?
-      puts "Not including any external assets"
-      next
+    if url_template.blank? || target_path.blank?
+      raise 'usage: rake deploy:download_external_assets[target_path,url_template[,"user:password"][,branch]]'
     end
 
+    FileUtils.mkdir_p target_path
+
     urls = [current_branch, Settings.default_asset_branch, 'master'].map do |branch|
       branch.present? ? url_template.gsub("<branch>", branch) : nil
     end.compact
@@ -39,7 +53,7 @@ namespace :deploy do
     # Download tar file --------------------------
     errors, response = [], nil
     urls.each do |url|
-      puts "Including external assets from #{url}"
+      puts "Downloading external assets from #{url}"
       response = HTTParty.get url, http_options
       break if response.success?
       errors << "HTTP error while trying to download assets from #{url}: #{response.inspect}"
@@ -59,11 +73,16 @@ namespace :deploy do
     result = system(untar_cmd)
     raise "Error running `#{untar_cmd}`" unless result
 
-    assets_dir = `tar -tf #{tar_file_path} | head -n 1`
-    assets_dir.strip!
-    assets_dir_full_path = Rails.root.join("tmp", assets_dir)
+    tmp_assets_dir = `tar -tf #{tar_file_path} | head -n 1`
+    tmp_assets_dir.strip!
+
+    # Move assets to to target_path
+    files_to_mv = Dir.glob(Rails.root.join("tmp", tmp_assets_dir, "*"))
+    FileUtils.mv files_to_mv, target_path
+  end
 
-    cmd = "RAILS_ENV=#{Rails.env} EXTERNAL_ASSETS_PATHS=#{assets_dir_full_path} rake assets:precompile"
+  task :precompile_assets, [:external_asset_paths] => :environment do |t, args|
+    cmd = "RAILS_ENV=#{Rails.env} EXTERNAL_ASSET_PATHS=#{args[:external_asset_paths]} rake assets:precompile"
     puts "Running: #{cmd}"
     exec(cmd)
   end

spec/requests/liquid_rendering_spec.rb

@@ -21,22 +21,4 @@
       end
     end
   end
-
-  describe 'rendering sidebars' do
-    before :each do
-      LiquidMarkupSeeder.seed(quiet: true) # transactional fixtures nuke em every test :/
-    end
-
-    it 'renders the fundraiser sidebar' do
-      page = create :page, liquid_layout: LiquidLayout.find_by(title: 'Generic Fundraiser')
-      get "/pages/#{page.id}"
-      expect(response.body).to include('<div class="fundraiser-bar__content">')
-    end
-
-    it 'renders the petition sidebar' do
-      page = create :page, liquid_layout: LiquidLayout.find_by(title: 'Generic Petition')
-      get "/pages/#{page.id}"
-      expect(response.body).to include('<div class="petition-bar__content">')
-    end
-  end
 end