Merge branch 'master' into rss_agent_logs_exact_feed_on_error

.env.example

@@ -167,7 +167,7 @@ TIMEZONE="Pacific Time (US & Canada)"
 FAILED_JOBS_TO_KEEP=100
 
 # Maximum runtime of background jobs in minutes
-DELAYED_JOB_MAX_RUNTIME=20
+DELAYED_JOB_MAX_RUNTIME=2
 
 # Amount of seconds for delayed_job to sleep before checking for new jobs
 DELAYED_JOB_SLEEP_DELAY=10

CHANGES.md

@@ -1,12 +1,19 @@
 # Changes
 
+* Jul 30, 2015   - RssAgent can configure the order of events created via `events_order`.
+* Jul 29, 2015   - WebsiteAgent can configure the order of events created via `events_order`.
+* Jul 29, 2015   - DataOutputAgent can configure the order of events in the output via `events_order`.
 * Jul 20, 2015   - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
 * Jul 20, 2015   - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
+* Jul 8, 2015    - DataOutputAgent supports feed icon, and a new template variable `events`.
 * Jul 1, 2015    - DeDuplicationAgent properly handles destruction of memory.
 * Jun 26, 2015   - Add `max_events_per_run` to RssAgent.
 * Jun 19, 2015   - Add `url_from_event` to WebsiteAgent.
 * Jun 17, 2015   - RssAgent emits events for new feed items in chronological order.
+* Jun 17, 2015   - Liquid filter `unescape` added.
+* Jun 17, 2015   - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
 * Jun 15, 2015   - Liquid filter `uri_expand` added.
+* Jun 13, 2015   - Liquid templating engine is upgraded to version 3.
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

Gemfile

@@ -1,5 +1,8 @@
 source 'https://rubygems.org'
 
+# Ruby 2.0 is the minimum requirement
+ruby ['2.0.0', RUBY_VERSION].max
+
 # Optional libraries.  To conserve RAM, comment out any that you don't need,
 # then run `bundle` and commit the updated Gemfile and Gemfile.lock.
 gem 'twilio-ruby', '~> 3.11.5'    # TwilioAgent

Gemfile.lock

@@ -235,7 +235,7 @@ GEM
     launchy (2.4.2)
       addressable (~> 2.3)
     libv8 (3.16.14.7)
-    liquid (3.0.3)
+    liquid (3.0.6)
     listen (2.7.9)
       celluloid (>= 0.15.2)
       rb-fsevent (>= 0.9.3)
@@ -580,6 +580,3 @@ DEPENDENCIES
   weibo_2!
   wunderground (~> 1.2.0)
   xmpp4r (~> 0.5.6)
-
-BUNDLED WITH
-   1.10.5

README.md

@@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif
 
 ## Deployment
 
-[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
+Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup.  Be sure to click 'View it' after launch!)
 
-Huginn can run on Heroku for free!  Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
+Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
+
+Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
 
 ### Optional Setup
 

app/concerns/dry_runnable.rb

@@ -1,10 +1,8 @@
 module DryRunnable
-  def dry_run!
-    readonly!
+  extend ActiveSupport::Concern
 
-    class << self
-      prepend Sandbox
-    end
+  def dry_run!
+    @dry_run = true
 
     log = StringIO.new
     @dry_run_logger = Logger.new(log)
@@ -14,6 +12,7 @@ class << self
 
     begin
       raise "#{short_type} does not support dry-run" unless can_dry_run?
+      readonly!
       check
     rescue => e
       error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
@@ -23,28 +22,38 @@ class << self
       memory: memory,
       log: log.string,
     )
+  ensure
+    @dry_run = false
   end
 
   def dry_run?
-    is_a? Sandbox
+    !!@dry_run
+  end
+
+  included do
+    prepend Wrapper
   end
 
-  module Sandbox
+  module Wrapper
     attr_accessor :results
 
     def logger
+      return super unless dry_run?
       @dry_run_logger
     end
 
-    def save
-      valid?
+    def save(options = {})
+      return super unless dry_run?
+      perform_validations(options)
     end
 
-    def save!
-      save or raise ActiveRecord::RecordNotSaved
+    def save!(options = {})
+      return super unless dry_run?
+      save(options) or raise_record_invalid
     end
 
     def log(message, options = {})
+      return super unless dry_run?
       case options[:level] || 3
       when 0..2
         sev = Logger::DEBUG
@@ -57,10 +66,12 @@ def log(message, options = {})
       logger.log(sev, message)
     end
 
-    def create_event(event_hash)
+    def create_event(event)
+      return super unless dry_run?
       if can_create_events?
-        @dry_run_results[:events] << event_hash[:payload]
-        events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
+        event = build_event(event)
+        @dry_run_results[:events] << event.payload
+        event
       else
         error "This Agent cannot create events!"
       end

app/concerns/sortable_events.rb

@@ -0,0 +1,161 @@
+module SortableEvents
+  extend ActiveSupport::Concern
+
+  included do
+    validate :validate_events_order
+  end
+
+  def description_events_order(*args)
+    self.class.description_events_order(*args)
+  end
+
+  module ClassMethods
+    def can_order_created_events!
+      raise if cannot_create_events?
+      prepend AutomaticSorter
+    end
+
+    def can_order_created_events?
+      include? AutomaticSorter
+    end
+
+    def cannot_order_created_events?
+      !can_order_created_events?
+    end
+
+    def description_events_order(events = 'events created in each run')
+      <<-MD.lstrip
+        To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
+
+        * _expression_ is a Liquid template to generate a string to be used as sort key.
+
+        * _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
+
+        * _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
+
+        Sort keys listed earlier take precedence over ones listed later.  For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
+
+        Sorting is done stably, so even if all events have the same set of sort key values the original order is retained.  Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
+      MD
+    end
+  end
+
+  def can_order_created_events?
+    self.class.can_order_created_events?
+  end
+
+  def cannot_order_created_events?
+    self.class.cannot_order_created_events?
+  end
+
+  def events_order
+    options['events_order']
+  end
+
+  module AutomaticSorter
+    def check
+      return super unless events_order
+      sorting_events do
+        super
+      end
+    end
+
+    def receive(incoming_events)
+      return super unless events_order
+      # incoming events should be processed sequentially
+      incoming_events.each do |event|
+        sorting_events do
+          super([event])
+        end
+      end
+    end
+
+    def create_event(event)
+      if @sortable_events
+        event = build_event(event)
+        @sortable_events << event
+        event
+      else
+        super
+      end
+    end
+
+    private
+
+    def sorting_events(&block)
+      @sortable_events = []
+      yield
+    ensure
+      events, @sortable_events = @sortable_events, nil
+      sort_events(events).each do |event|
+        create_event(event)
+      end
+    end
+  end
+
+  private
+
+  EXPRESSION_PARSER = {
+    'string' => ->string { string },
+    'number' => ->string { string.to_f },
+    'time'   => ->string { Time.zone.parse(string) },
+  }
+  EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
+
+  def validate_events_order
+    case order_by = events_order
+    when nil
+    when Array
+      # Each tuple may be either [expression, type, desc] or just
+      # expression.
+      order_by.each do |expression, type, desc|
+        case expression
+        when String
+          # ok
+        else
+          errors.add(:base, "first element of each events_order tuple must be a Liquid template")
+          break
+        end
+        case type
+        when nil, *EXPRESSION_TYPES
+          # ok
+        else
+          errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
+          break
+        end
+        if !desc.nil? && boolify(desc).nil?
+          errors.add(:base, "third element of each events_order tuple must be a boolean value")
+          break
+        end
+      end
+    else
+      errors.add(:base, "events_order must be an array of arrays")
+    end
+  end
+
+  # Sort given events in order specified by the "events_order" option
+  def sort_events(events)
+    order_by = events_order.presence or
+      return events
+
+    orders = order_by.map { |_, _, desc = false| boolify(desc) }
+
+    Utils.sort_tuples!(
+      events.map.with_index { |event, index|
+        interpolate_with(event) {
+          interpolation_context['_index_'] = index
+          order_by.map { |expression, type, _|
+            string = interpolate_string(expression)
+            begin
+              EXPRESSION_PARSER[type || 'string'.freeze][string]
+            rescue
+              error "Cannot parse #{string.inspect} as #{type}; treating it as string"
+              string
+            end
+          }
+        } << index << event  # index is to make sorting stable
+      },
+      orders
+    ).collect!(&:last)
+  end
+end

app/helpers/application_helper.rb

@@ -80,6 +80,7 @@ def service_label_text(service)
   end
 
   def service_label(service)
+    return if service.nil?
     content_tag :span, [
       omniauth_provider_icon(service.provider),
       service_label_text(service)

app/models/agent.rb

@@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base
   include HasGuid
   include LiquidDroppable
   include DryRunnable
+  include SortableEvents
 
   markdown_class_attributes :description, :event_description
 
@@ -104,12 +105,19 @@ def working?
     raise "Implement me in your subclass"
   end
 
-  def create_event(attrs)
+  def build_event(event)
+    event = events.build(event) if event.is_a?(Hash)
+    event.agent = self
+    event.user = user
+    event.expires_at ||= new_event_expiration_date
+    event
+  end
+
+  def create_event(event)
     if can_create_events?
-      events.create!({
-         :user => user,
-         :expires_at => new_event_expiration_date
-      }.merge(attrs))
+      event = build_event(event)
+      event.save!
+      event
     else
       error "This Agent cannot create events!"
     end

app/models/agents/data_output_agent.rb

@@ -40,11 +40,15 @@ class DataOutputAgent < Agent
               "_contents": "tag contents (can be an object for nesting)"
             }
 
+        # Ordering events in the output
+
+        #{description_events_order('events in the output')}
+
         # Liquid Templating
 
         In Liquid templating, the following variable is available:
 
-        * `events`: An array of events being output, sorted in descending order up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
+        * `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number.  For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
 
       MD
     end
@@ -134,7 +138,7 @@ def receive_web_request(params, method, format)
         end
       end
 
-      source_events = received_events.order(id: :desc).limit(events_to_show).to_a
+      source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
 
       interpolation_context.stack do
         interpolation_context['events'] = source_events