User agent renamed to avoid blacklisting by CDNs

- fixed issue 1695
- remove /comments to reduce spam
- updated delayed job config

Gemfile

@@ -139,3 +139,4 @@ gem 'sinatra', :require => nil
 gem 'sidekiq-failures'
 gem 'rails-i18n', '~> 3.0.0'
 gem 'eventmachine', '~> 1.2.5'
+gem 'daemons'

Gemfile.lock

@@ -143,6 +143,7 @@ GEM
       nokogiri (~> 1.5)
       railties (>= 3, < 5.1)
     cucumber-wire (0.0.1)
+    daemons (1.2.6)
     database_cleaner (1.6.2)
     datapackage (0.0.4)
       colorize
@@ -736,6 +737,7 @@ DEPENDENCIES
   coveralls
   csvlint!
   cucumber-rails
+  daemons
   data_kitten!
   database_cleaner
   delayed-plugins-airbrake

Procfile

@@ -1,2 +1,3 @@
 web: bundle exec puma -C config/puma.rb
 worker: bundle exec sidekiq -c 3
+delayed_job: bundle exec rake jobs:work

app/helpers/application_helper.rb

@@ -6,11 +6,6 @@ def admin?
     current_user && current_user.admin?
   end
 
-  def comment_link topic, message = 'comment', title = topic, question_id = nil
-    content = "<i class=\"icon-comments\"></i> <span>#{message}</span>"
-    link_to content.html_safe, comment_path(topic: topic, back: request.original_fullpath, title: title, question_id: question_id), class: 'link-comment'
-  end
-
   def embed_protocol
     Rails.env.production? ? 'https://' : request.protocol
   end

app/views/partials/_question.html.haml

@@ -142,5 +142,3 @@
             - if (q.help_text_more_url)
               %a{:href => q.help_text_more_url}
                 = t('surveyor.question_comment_read_more')
-        - if q.discussion_topic
-          = comment_link q.discussion_topic, t('surveyor.question_comment'), q.text, q.id

app/views/response_sets/_response_set.html.haml

@@ -37,9 +37,7 @@
                 %span.label.odc-popover
                   %span.icon.icon-flag
                   %span.odc-popover-content
-                    Do you think this data is incorrect?
-                    = link_to comment_path(topic: "certificates/#{response_set.certificate.id}/#{response.question.reference_identifier}", back: request.original_fullpath, title: "#{response.question.statement_text} #{response_text} ") do
-                      %span.btn.btn-mini.btn-info Let us know <i class="icon icon-chevron-right"></i>
+                    Do you think this data is incorrect? Let us know</i>
 
               - if state
                 %p.error

app/views/surveyor/requirements.html.haml

@@ -51,7 +51,7 @@
                       %p= markdown(field.help_text || '')
                     .span6.offset2
                       =button_to t('improvements.buttons.understood'), resume_path(field), method: 'post', class: 'btn btn-primary btn-large'
-                      %a.btn.btn-primary.btn-large.btn-negative{:href => comment_path(topic: field.discussion_topic, back: request.original_fullpath, title: field.text, question_id: field.id)}
+                      %a.btn.btn-primary.btn-large.btn-negative{:href => '/comment' }
                         =t('improvements.buttons.misunderstood')
               %hr.heavy
 
@@ -71,6 +71,6 @@
                       %p= markdown(answer.try(:help_text) || question.try(:help_text) || '')
                     .span6.offset2
                       =button_to t('improvements.buttons.understood'), resume_path(question), method: 'post', class: 'btn btn-primary btn-large'
-                      %a.btn.btn-primary.btn-large.btn-negative{:href => comment_path(topic: question.discussion_topic, back: request.original_fullpath, title: question.text, question_id: question.id)}
+                      %a.btn.btn-primary.btn-large.btn-negative{:href => '/comment'}
                         =t('improvements.buttons.misunderstood')
               %hr.heavy

app/views/surveyor/start.html.haml

@@ -56,7 +56,7 @@
             .lead
               = t('surveys.dataset_description_html')
           .span4
-            %button.btn.btn-large.btn-primary.submit{data:{toggle: 'popover', placement: 'bottom', content: t('surveys.url_problem_error'), error: t('surveys.url_unknown_error')}}
+            %button.btn.btn-large.btn-primary.submit{data:{toggle: 'popover', placement: 'bottom', content: t('surveys.please_wait_autocomplete'), error: t('surveys.url_unknown_error')}}
               %i.icon-loading.icon-spin.icon-refresh
               %i.icon-exclamation-sign
               = t('surveys.check_url_button')

config/initializers/delayed_job.rb

@@ -0,0 +1,5 @@
+Delayed::Worker.destroy_failed_jobs = false
+Delayed::Worker.logger = Logger.new(File.join(Rails.root, 'log', 'delayed_job.log'))
+Delayed::Worker.default_log_level = 'debug'.freeze
+Delayed::Worker.max_run_time = 8.hours
+Delayed::Worker.max_attempts = 1

config/locales/website_en.yml

@@ -235,6 +235,7 @@ en:
     skip_this_button: Skip this
     edit_button: Edit
     url_problem_error: There was a problem with your URL, please check, or fill out the explanation below
+    please_wait_autocomplete: Please wait while we attempt to autocomplete the questionnaire using the URL you provided
     url_unknown_error: Sorry, an error occurred. Please try again.
     dataset_description_html: |
       <p>Link to the dataset you are certifying. By dataset we mean

lib/odibot.rb

@@ -1,30 +1,30 @@
 class ODIBot
   include HTTParty
 
-  USER_AGENT = "ODICertBot 1.0 (+https://certificates.theodi.org/)"
+  USER_AGENT = "ODICertCheck 1.1 (+https://certificates.theodi.org/)"
   NETWORK_ERRORS = [
-    EOFError,
-    SocketError,
-    Errno::ETIMEDOUT,
-    Errno::ECONNREFUSED,
-    Errno::ECONNRESET,
-    Errno::EHOSTUNREACH,
-    OpenSSL::SSL::SSLError,
-    Timeout::Error,
-    Net::ReadTimeout
+      EOFError,
+      SocketError,
+      Errno::ETIMEDOUT,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      OpenSSL::SSL::SSLError,
+      Timeout::Error,
+      Net::ReadTimeout
   ]
   HTTP_ERRORS = [
-    HTTParty::RedirectionTooDeep,
-    URI::InvalidURIError,
-    Zlib::DataError
+      HTTParty::RedirectionTooDeep,
+      URI::InvalidURIError,
+      Zlib::DataError
   ]
 
   def self.valid?(url)
     new(url).valid?
   end
 
   def initialize(url)
-    @options = { headers: {"User-Agent" => USER_AGENT } }
+    @options = {headers: {"User-Agent" => USER_AGENT}}
     @url = url
   end
 
@@ -41,8 +41,7 @@ def response_code
 
   def is_http_url?
     if uri.kind_of?(URI::HTTP)
-      hostname = uri.hostname
-      hostname.present? && hostname.include?('.')
+      is_hostname_valid?
     else
       false
     end
@@ -78,11 +77,32 @@ def check_ckan_endpoint
           r = Net::HTTP.get_response(URI.parse(r.header['location']))
         end
         success = JSON.parse(r.body).has_key?("version") if r.code.eql?("200")
-      rescue 
+      rescue
         success = false
       end
+    else
+      success = check_ckan_endpoint_status
     end
-    { success: success, redirect: redirect }
+    {success: success, redirect: redirect}
+  end
+
+  def check_ckan_endpoint_status
+    success = false
+    if is_hostname_valid?
+      begin
+        status_uri = uri.clone
+        status_uri.path += '/util/status'
+        r = Net::HTTP.get_response(status_uri)
+        success = JSON.parse(r.body).has_key?("ckan_version") if r.code.eql?("200")
+      rescue
+        success = false
+      end
+    end
+    success
+  end
+
+  def is_hostname_valid?
+    uri.hostname.present? && uri.hostname.include?('.')
   end
 
   def self.handle_errors(return_value)
@@ -91,9 +111,10 @@ def self.handle_errors(return_value)
     return return_value
   end
 
-private
+  private
+
   def escape_unsafe_chars(url)
-    url.gsub(URI::UNSAFE) { |c| URI.escape(c) }
+    url.gsub(URI::UNSAFE) {|c| URI.escape(c)}
   end
 
 end

lib/tasks/odc.rake

@@ -109,6 +109,7 @@ namespace :odc do
 
   end
 
+  desc "create a csv export from all of the current published datasets"
   task :generate_dataset_csv => :environment do
     Delayed::Job.enqueue CSVExport, { :priority => 5, :run_at => 10.minutes.from_now.utc }
   end

test/unit/odibot_test.rb

@@ -144,4 +144,28 @@ class ODIBotTest < ActiveSupport::TestCase
     assert_equal false, result[:success]
   end
 
+  test "successfully checks valid CKAN API which returns a valid status" do
+    stub_request(:any, "http://www.example.com/api").to_return(status: 404)
+    stub_request(:any, "http://www.example.com/api/util/status").to_return(:body => {ckan_version: "2.8"}.to_json, status: 200)
+    bot = ODIBot.new("http://www.example.com/api")
+    result = bot.check_ckan_endpoint
+    assert_equal true, result[:success]
+  end
+
+  test "handles valid CKAN API which fails to return an 'OK' status" do
+    stub_request(:any, "http://www.example.com/api").to_return(status: 404)
+    stub_request(:any, "http://www.example.com/api/util/status").to_return(status: 404)
+    bot = ODIBot.new("http://www.example.com/api")
+    result = bot.check_ckan_endpoint
+    assert_equal false, result[:success]
+  end
+
+  test "handles valid CKAN API which fails to returns a '200' status but without the required attribute(s)" do
+    stub_request(:any, "http://www.example.com/api").to_return(status: 404)
+    stub_request(:any, "http://www.example.com/api/util/status").to_return(:body => {foo_bar: "2.8"}.to_json, status: 200)
+    bot = ODIBot.new("http://www.example.com/api")
+    result = bot.check_ckan_endpoint
+    assert_equal false, result[:success]
+  end
+
 end