@@ -232,8 +232,11 @@ DESCRIPTION :priority => any number, including negative ones. default is zero. - :tag => a tag added to the job. simply makes searching easier. + :tag => a tag added to the job. makes searching easier, and can also be + used as filters in runners via --only-tag and --except-tag options. + :dependency_id => a job id which has to be run before this job + :env => a hash specifying any additional environment vars the background process should have. @@ -253,6 +256,11 @@ DESCRIPTION :tag => 'dynamic ruby code' jobs Bj.submit array_of_commands, :priority => 451 + + # Dependency example + + primary_job = Bj.submit './script/runner lib/jobs/primary.rb' + Bj.submit './script/runner lib/jobs/post_primary.rb', :dependency_id => primary_job.first.bj_job_id when jobs are run, they are run in RAILS_ROOT. various attributes are available *only* once the job has finished. you can check whether or not a @@ -297,6 +305,10 @@ PARAMETERS --log=log, -l (0 ~> log=STDERR) set the logfile --help, -h + --only-tag=tag + Only run jobs tagged with this tag + --except-tag=tag + Run all jobs except those tagged with this tag AUTHOR ara.t.howard@gmail.com README @@ -378,6 +378,14 @@ txt argument :required } option("--daemon"){} + option("--only-tag"){ + argument :required + description "Only execute jobs with this tag name" + } + option("--except-tag"){ + argument :required + description "Excute all jobs which don't have this tag name" + } def run options = {} @@ -388,7 +396,7 @@ txt end =end - %w[ forever ppid wait limit ].each do |key| + %w[ forever ppid wait limit only-tag except-tag ].each do |key| options[key.to_sym] = param[key].value if param[key].given? end bin/bj @@ -94,13 +94,11 @@ class Bj def key ppid = 0 ppid ||= 0 - #"#{ Bj.rails_env }.#{ ppid }.pid" - "runner.#{ ppid }.pid" + "#{ Bj.rails_env }.#{ ppid }.pid" end def no_tickle_key - # "#{ Bj.rails_env }.no_tickle" - "no_tickle" + "#{ Bj.rails_env }.no_tickle" end def start options = {} @@ -156,7 +154,9 @@ class Bj wait = options[:wait] || 42 limit = options[:limit] forever = options[:forever] - + only_tag = options["only-tag".to_sym] + except_tag = options["except-tag".to_sym] + limit = false if forever wait = Integer wait loopno = 0 @@ -186,19 +186,29 @@ class Bj Bj.transaction(options) do now = Time.now - - job = Bj::Table::Job.find :first, - :conditions => ["state = ? and submitted_at <= ?", "pending", now], - :order => "priority DESC, submitted_at ASC", - :limit => 1, - :lock => true + + conditions = "" + conditions += "and tag = '#{only_tag}'" if only_tag + conditions += "and tag != '#{except_tag}'" if except_tag + + jobs = Bj::Table::Job.find :all, + :conditions => ["state = ? and submitted_at <= ? #{conditions}", "pending", now], + :order => "priority DESC, submitted_at ASC", + :lock => true + + job = nil + jobs.each do |j| + job = j and break if j.dependency_id.nil? + + dependency = Bj::Table::Job.find j.dependency_id + job = j and break if dependency.state == 'finished' + end throw :no_jobs unless job - Bj.logger.info{ "#{ job.title } - started" } command = job.command - env = job.env ? YAML.load(job.env) : {} + env = job.env || {} stdin = job.stdin || '' stdout = job.stdout || '' stderr = job.stderr || '' lib/bj/runner.rb @@ -102,7 +102,8 @@ class Bj t.column "priority" , :integer t.column "tag" , :text t.column "is_restartable" , :integer - + t.column "dependency_id" , :integer + t.column "submitter" , :text t.column "runner" , :text t.column "pid" , :integer lib/bj/table.rb 5d9a4efcdd9b754e9f6cc04fd21ab6cf0d02a38f Michael Murray mdmurray@gmail.com http://github.com/howcast/backgroundjob/commit/c25a6a6e1cf78fc02a2335ebf0f454715a4a3ac9 c25a6a6e1cf78fc02a2335ebf0f454715a4a3ac9 2008-05-20T23:03:56-07:00 2008-05-20T23:03:56-07:00 Adds more options for distributed bj: - Workers which respect tags: Workers can now be started with --only-tag=tag and --except-tag=tag options which will allow for them to only run jobs with these conditions. This is useful if you have specialized worker environments where certain workers can only run or not run particular jobs. - Dependencies: With a distributed system it may be important that some jobs are run consecutively, with the tag filters it can get even harder to avoid deadlock situations in the job queue. This enhancement allows jobs to be submitted with a dependency_id (another bj_job_id) which must be completed before the job being submitted is started. See the updated README for an example. f163692318cbadee4723657578e4a872a82a4762 Michael Murray mdmurray@gmail.com