Permalink
Browse files

Update Hombrew github URL (Closes #52)

Also replace sidetiq with sidekiq-scheduler and add sidekiq-unique-jobs
  • Loading branch information...
1 parent 4419ec6 commit 9262cb551a2d90d5f9be415415d82231bbb39c5e @zedtux committed Jul 10, 2016
View
@@ -14,7 +14,8 @@ gem 'slim-rails'
gem 'git'
gem 'appconfig', require: 'app_config'
gem 'sidekiq'
-gem 'sidetiq' # Recurring jobs
+gem 'sidekiq-scheduler' # Recurring jobs
+gem 'sidekiq-unique-jobs' # The missing unique jobs in sidekiq
gem 'sinatra', '>= 1.3.0', require: nil
gem 'newrelic_rpm'
gem 'nokogiri'
View
@@ -60,23 +60,6 @@ GEM
capybara-screenshot (1.0.12)
capybara (>= 1.0, < 3)
launchy
- celluloid (0.17.3)
- celluloid-essentials
- celluloid-extras
- celluloid-fsm
- celluloid-pool
- celluloid-supervision
- timers (>= 4.1.1)
- celluloid-essentials (0.20.5)
- timers (>= 4.1.1)
- celluloid-extras (0.20.5)
- timers (>= 4.1.1)
- celluloid-fsm (0.20.5)
- timers (>= 4.1.1)
- celluloid-pool (0.20.5)
- timers (>= 4.1.1)
- celluloid-supervision (0.20.5)
- timers (>= 4.1.1)
childprocess (0.5.9)
ffi (~> 1.0, >= 1.0.11)
chronic (0.10.2)
@@ -140,11 +123,9 @@ GEM
globalid (0.3.6)
activesupport (>= 4.1.0)
hashdiff (0.3.0)
- hitimes (1.2.3)
htmlentities (4.3.4)
hub (1.12.4)
i18n (0.7.0)
- ice_cube (0.13.3)
jquery-rails (4.1.1)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
@@ -265,6 +246,7 @@ GEM
unicode-display_width (~> 1.0, >= 1.0.1)
ruby-progressbar (1.7.5)
rubyzip (1.2.0)
+ rufus-scheduler (3.1.10)
safe_yaml (1.0.4)
selenium-webdriver (2.53.0)
childprocess (~> 0.5)
@@ -277,10 +259,15 @@ GEM
concurrent-ruby (~> 1.0)
connection_pool (~> 2.2, >= 2.2.0)
redis (~> 3.2, >= 3.2.1)
- sidetiq (0.7.0)
- celluloid (>= 0.17.3)
- ice_cube (~> 0.13.2)
- sidekiq (>= 4.0.0)
+ sidekiq-scheduler (2.0.8)
+ multi_json (~> 1)
+ redis (~> 3)
+ rufus-scheduler (~> 3.1.8)
+ sidekiq (>= 3)
+ tilt (>= 1.4.0)
+ sidekiq-unique-jobs (4.0.17)
+ sidekiq (>= 2.6)
+ thor
simplecov (0.11.2)
docile (~> 1.1.0)
json (~> 1.8)
@@ -316,8 +303,6 @@ GEM
thread_safe (0.3.5)
tilt (2.0.2)
timecop (0.8.1)
- timers (4.1.1)
- hitimes
tins (1.6.0)
turbolinks (2.5.3)
coffee-rails
@@ -382,7 +367,8 @@ DEPENDENCIES
selenium-webdriver
shoulda-matchers
sidekiq
- sidetiq
+ sidekiq-scheduler
+ sidekiq-unique-jobs
sinatra (>= 1.3.0)
slim-rails
therubyracer
@@ -392,4 +378,4 @@ DEPENDENCIES
yard
BUNDLED WITH
- 1.11.2
+ 1.12.5
@@ -5,12 +5,10 @@
#
class HomebrewFormulaImportWorker
include Sidekiq::Worker
- include Sidetiq::Schedulable
- sidekiq_options backtrace: true
- attr_accessor :formula
+ sidekiq_options backtrace: true, retry: false, unique: :until_executed
- recurrence { daily }
+ attr_accessor :formula
#
# Create new Homebrew::Formula if missing otherwise update the existing one
@@ -31,7 +29,7 @@ def formulas_path
File.join(
AppConfig.homebrew.git_repository.location,
AppConfig.homebrew.git_repository.name,
- 'Library', 'Formula', '*.rb'
+ 'Formula', '*.rb'
)
end
@@ -42,18 +40,17 @@ def import_formulas
formulas = Dir[formulas_path]
formulas.each { |path| create_or_update_formula_from(path) }
- Rails.logger.info <<-eos
- Iterated over #{formulas.size} formulas and #{Homebrew::Formula.count}
- in database including #{Homebrew::Formula.externals.count}
- external formulas.
- eos
+ Rails.logger.info "Iterated over #{formulas.size} formulas and " \
+ "#{Homebrew::Formula.count} in database including " \
+ "#{Homebrew::Formula.externals.count} external formulas."
end
#
# Get the Homebrew source code from Github
# and create/update the Homebrew::Formula
#
def perform
+ Rails.logger.info 'Starting new import ...'
@import = Import.create!(success: true)
import_formulas
@@ -62,15 +59,14 @@ def perform
@import.success = false
raise
rescue ActiveRecord::RecordInvalid => error
- Rails.logger.warn 'Unable to create a new import : ' \
- "#{error.message}"
+ Rails.logger.warn "Unable to create a new import : #{error.message}"
ensure
@import.ended_at = Time.now
if @import.save
DynamicSitemaps.generate_sitemap
else
- Rails.logger.warn 'Unable to update the import with ID ' \
- "#{@import.id} : #{@import.errors.full_message}"
+ Rails.logger.warn "Unable to update the import with ID #{@import.id} : " \
+ "#{@import.errors.full_message}"
end
end
@@ -5,7 +5,7 @@ brewformulas:
keywords: 'MAC, Apple, Homebrew, brew, mxcl, Formula, Formulas, Formulae, search, description'
homebrew:
git_repository:
- url: https://github.com/Homebrew/homebrew.git
+ url: https://github.com/Homebrew/homebrew-core.git
name: homebrew
location: <%= File.join(Rails.root, 'cache') %>
redis:
View
@@ -1,7 +1,7 @@
postgresql: &postgresql
adapter: postgresql
- host: <%= ENV['POSTGRES_PORT_5432_TCP_ADDR'] || ENV['BREWFORMULASORG_POSTGRESQL_1_PORT_5432_TCP_ADDR'] || 'localhost' %>
- port: <%= ENV['POSTGRES_PORT_5432_TCP_PORT'] || ENV['BREWFORMULASORG_POSTGRESQL_1_PORT_5432_TCP_PORT'] || 5432 %>
+ host: <%= ENV['POSTGRES_1_PORT_5432_TCP_ADDR'] || ENV['BREWFORMULASORG_POSTGRESQL_1_PORT_5432_TCP_ADDR'] || 'localhost' %>
+ port: <%= ENV['POSTGRES_1_PORT_5432_TCP_PORT'] || ENV['BREWFORMULASORG_POSTGRESQL_1_PORT_5432_TCP_PORT'] || 5432 %>
username: <%= ENV['POSTGRESQL_USER'] || 'brewformulas' %>
password: <%= ENV['POSTGRESQL_PASSWORD'] %>
template: <%= ENV['POSTGRESQL_TEMPLATE'] || 'template0' %>
@@ -1,5 +0,0 @@
-# When running on Tutum.co platform, redirect the logs to the standard output
-# so that they are accessible from the Tutum.co dashboard
-Rails.logger = Logger.new(STDOUT) if ENV.key?('TUTUM')
-
-Rails.logger.info 'Tutum initialization done.'
@@ -34,12 +34,13 @@
Sidekiq.configure_server do |config|
config.redis = { url: redis_url, namespace: AppConfig.redis.try(:namespace) }
+ config.on(:startup) do
+ scheduler_file_path = File.expand_path('../../../config/scheduler.yml',
+ __FILE__)
+ Sidekiq.schedule = YAML.load_file(scheduler_file_path)
+ Sidekiq::Scheduler.reload_schedule!
+ end
end
Sidekiq.configure_client do |config|
config.redis = { url: redis_url, namespace: AppConfig.redis.try(:namespace) }
end
-
-Sidetiq.configure do |config|
- # When `true` uses UTC instead of local times (default: false).
- config.utc = false
-end
View
@@ -1,5 +1,5 @@
require 'sidekiq/web'
-require 'sidetiq/web'
+require 'sidekiq-scheduler/web'
BrewformulasOrg::Application.routes.draw do
mount Sidekiq::Web => '/sidekiq'
@@ -0,0 +1,3 @@
+fetch_homebrew_formulae:
+ cron: "0 1 * * *"
+ class: HomebrewFormulaImportWorker
View
@@ -1,4 +1,5 @@
---
:concurrency: 25
+:logfile: ./log/sidekiq.log
production:
:concurrency: 4
View
@@ -10,7 +10,9 @@ workers:
- redis:redis
environment:
- POSTGRESQL_USER=postgres
- - RUNNING_IN_DOCKER=true
+ - RAILS_ENV=development
+ volumes:
+ - .:/application/
bundle:
image: brewformulasorg_web
command: echo Bundler data container

0 comments on commit 9262cb5

Please sign in to comment.