Merge pull request #122 from alphagov/need-exporter-improvements

Need exporter improvements
alphagov · Feb 1, 2017 · 1eda9cc · 1eda9cc
2 parents de7ba20 + 06cb003
commit 1eda9cc
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 72 deletions.
diff --git a/Gemfile b/Gemfile
@@ -9,7 +9,7 @@ gem 'plek', '~> 1.11'
 if ENV['API_DEV']
   gem 'gds-api-adapters', path: '../gds-api-adapters'
 else
-  gem 'gds-api-adapters', '26.2.0'
+  gem 'gds-api-adapters', git: 'https://github.com/alphagov/gds-api-adapters.git', branch: 'add-import-endpoint-adapter'
 end
 
 gem 'mongoid_rails_migrations', '1.1.0'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,3 +1,16 @@
+GIT
+  remote: https://github.com/alphagov/gds-api-adapters.git
+  revision: 6b859b45f5e5186683803031ad8aa3801c4f6083
+  branch: add-import-endpoint-adapter
+  specs:
+    gds-api-adapters (39.1.0)
+      link_header
+      lrucache (~> 0.1.1)
+      null_logger
+      plek (>= 1.9.0)
+      rack-cache
+      rest-client (~> 2.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -53,7 +66,7 @@ GEM
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     database_cleaner (1.5.1)
-    domain_name (0.5.25)
+    domain_name (0.5.20161129)
       unf (>= 0.0.5, < 1.0.0)
     elasticsearch (1.0.14)
       elasticsearch-api (= 1.0.14)
@@ -71,13 +84,6 @@ GEM
       railties (>= 3.0.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    gds-api-adapters (26.2.0)
-      link_header
-      lrucache (~> 0.1.1)
-      null_logger
-      plek
-      rack-cache
-      rest-client (~> 1.8.0)
     gds-sso (11.2.0)
       multi_json (~> 1.0)
       oauth2 (~> 1.0)
@@ -93,7 +99,7 @@ GEM
       rubocop (~> 0.32)
     hashdiff (0.2.3)
     hashie (3.4.3)
-    http-cookie (1.0.2)
+    http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (0.7.0)
     json (1.8.3)
@@ -167,7 +173,7 @@ GEM
     rack (1.6.4)
     rack-accept (0.4.5)
       rack (>= 0.4)
-    rack-cache (1.5.1)
+    rack-cache (1.7.0)
       rack (>= 0.4)
     rack-test (0.6.3)
       rack (>= 1.0)
@@ -199,10 +205,10 @@ GEM
     raindrops (0.15.0)
     rake (10.5.0)
     redis (3.2.2)
-    rest-client (1.8.0)
+    rest-client (2.0.0)
       http-cookie (>= 1.0.2, < 2.0)
-      mime-types (>= 1.16, < 3.0)
-      netrc (~> 0.7)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
     rubocop (0.35.1)
       astrolabe (~> 1.3)
       parser (>= 2.2.3.0, < 3.0)
@@ -234,7 +240,7 @@ GEM
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.1)
+    unf_ext (0.0.7.2)
     unicorn (5.0.1)
       kgio (~> 2.6)
       rack
@@ -258,7 +264,7 @@ DEPENDENCIES
   database_cleaner (= 1.5.1)
   elasticsearch (= 1.0.14)
   factory_girl_rails (= 4.5.0)
-  gds-api-adapters (= 26.2.0)
+  gds-api-adapters!
   gds-sso (= 11.2.0)
   govuk-lint (~> 0.4)
   kaminari (= 0.16.3)
@@ -278,4 +284,4 @@ DEPENDENCIES
   whenever (= 0.9.4)
 
 BUNDLED WITH
-   1.10.6
+   1.13.7
diff --git a/lib/needs_exporter.rb b/lib/needs_exporter.rb
@@ -5,42 +5,51 @@
 class NeedsExporter
   def initialize
     @needs = Need.all
-    @api_client = GdsApi::PublishingApiV2.new(Plek.find('publishing-api'))
+    @api_client = GdsApi::PublishingApiV2.new(
+      Plek.find('publishing-api'),
+      bearer_token: ENV["PUBLISHING_API_BEARER_TOKEN"] || "example",
+      timeout: 30
+    )
     @slugs = Set.new
   end
 
   def run
-    @needs.each_with_index do |need, index|
-      export(need, index)
+    count = @needs.count
+    @needs.order_by(:_id.asc).each_with_index do |need, index|
+      export(need, index, count)
     end
   end
 
 private
 
-  def export(need, index)
+  def export(need, index, count)
     slug = generate_slug(need)
-    need_revision_groups = need_revision_groups(need.revisions)
-    snapshots = need_revision_groups.map { |nrg| present_need_revision_group(nrg, slug)}
+    need_revision_groups = need_revision_groups(need.revisions.sort_by(&:created_at))
+    snapshots = need_revision_groups.map { |nrg| present_need_revision_group(need.id, nrg, slug)}
     compute_superseded_needs(snapshots)
-    @api_client.import(need.content_id, snapshots)
+    @api_client.import(need.content_id, "en", snapshots)
     links = present_links(need)
     @api_client.patch_links(need.content_id, links: links)
-    p "#{index}/#{@needs.count}"
-    p "exported #{slug}"
+
+    padding = Math.log10(count).ceil
+    puts format("%#{padding}d/#{count} exported #{slug}", index + 1)
   end
 
-  def present_need_revision_group(need_revision_group, slug)
+  def present_need_revision_group(need_id, need_revision_group, slug)
     states = need_revision_group.map do |nr|
       map_to_publishing_api_state(nr, slug)
     end
 
+    last_snapshot = need_revision_group.last.snapshot
+    role, goal, benefit =
+      last_snapshot.values_at("role", "goal", "benefit").map(&:strip)
+
     {
-       title: need_revision_group.last.snapshot["benefit"],
+       title: "As a #{role}, I need to #{goal}, so that #{benefit} (#{need_id})",
        publishing_app: "need-api",
        schema_name: "need",
        document_type: "need",
        rendering_app: "info-frontend",
-       locale: "en",
        base_path: "/needs/#{slug}",
        states: states,
        routes: [{
@@ -69,11 +78,6 @@ def present_details(need_revision)
     details = {}
     need_revision.snapshot.each do |key, value|
       if should_not_be_in_details(key, value)
-        next
-      elsif key == "status"
-        details["status"] = value["description"]
-      elsif key.start_with?("yearly")
-        details["#{key}"] = value.to_s
       else
         details["#{key}"] = value
       end
@@ -87,16 +91,13 @@ def present_links(need)
       next unless is_a_link?(key) && value.present?
       if key == "organisation_ids"
         links["organisations"] = need.organisations.map(&:content_id)
-      elsif key == "duplicate_of"
-        links["related_items"] = related_needs(need).map(&:content_id)
       end
     end
     links
   end
 
   def is_a_link?(key)
-    key == "organisation_ids" ||
-      key == "duplicate_of"
+    key == "organisation_ids"
   end
 
   def should_not_be_in_details(key, value)
@@ -107,7 +108,7 @@ def should_not_be_in_details(key, value)
   end
 
   def generate_slug(need)
-    base_slug = need.benefit.parameterize
+    base_slug = need.goal.parameterize
     n = 0
     slug = ""
     loop do
@@ -124,18 +125,6 @@ def suffix(n)
     "-#{n}"
   end
 
-  def draft_already_in_list?(revisions_list)
-    status_list = []
-    revisions_list.each {|r| status_list << get_status(r)}
-    status_list.include?("proposed") || status_list.include?("not valid")
-  end
-
-  def published_already_in_list?(revisions_list)
-    status_list = []
-    revisions_list.each {|r| status_list << get_status(r)}
-    status_list.include?("valid") || status_list.include?("valid with conditions")
-  end
-
   def is_proposed?(need_revision)
     get_status(need_revision) == "proposed"
   end
@@ -148,28 +137,19 @@ def is_not_valid?(need_revision)
     get_status(need_revision) == "not valid"
   end
 
-  def related_needs(need)
-    Need.where(need_id: need.duplicate_of)
-  end
-
   def deprecated_fields
-    %w{monthly_user_contacts monthly_need_views currently_met in_scope out_of_scope_reason}
+    %w{status monthly_user_contacts monthly_need_views currently_met in_scope out_of_scope_reason duplicate_of}
   end
 
   def get_status(need_revision)
-    if includes_snapshot?(need_revision)
+    if snapshot_includes_status?(need_revision)
       need_revision.snapshot["status"]["description"]
     else
-      need_revision.need.status.description
+      "proposed" # Old revisions don't have a status, so just use "proposed"
     end
   end
 
-  def is_latest?(need_revision)
-    all_revisions = need_revision.need.revisions
-    all_revisions.select(&:created_at).max == need_revision
-  end
-
-  def includes_snapshot?(need_revision)
+  def snapshot_includes_status?(need_revision)
     need_revision.snapshot["status"] && need_revision.snapshot["status"]["description"]
   end
 
@@ -204,24 +184,39 @@ def get_revision_snapshot_status(snapshot)
   end
 
   def map_to_publishing_api_state(need_revision, slug)
-    if need_revision["duplicate_of"].present? && is_valid?(need_revision)
-      {
-        name: "unpublished",
-        type: "withdrawal",
-        date: need_revision["created_at"],
-        explanation: "This need is a duplicate_of the need [#{need_revision.need.benefit}](/needs/#{slug})"
-      }
+    if need_revision.snapshot["duplicate_of"].present?
+      begin
+        duplicate_need_id = need_revision.snapshot["duplicate_of"]
+        duplicate_content_id = Need.find(duplicate_need_id).content_id
+        {
+          name: "unpublished",
+          type: "withdrawal",
+          date: need_revision["created_at"],
+          explanation: "This need is a duplicate of: [embed:link:#{duplicate_content_id}]"
+        }
+      rescue Mongoid::Errors::DocumentNotFound
+        {
+          name: "unpublished",
+          type: "withdrawal",
+          date: need_revision["created_at"],
+          explanation: "This need is a duplicate of an unknown need (#{duplicate_need_id})"
+        }
+      end
     elsif is_proposed?(need_revision)
       {
         name: "draft",
         date: need_revision["created_at"]
       }
     elsif is_not_valid?(need_revision)
+      bulleted_reasons =
+        need_revision.snapshot["status"]["reasons"].map { |x| "* #{x}\n" }.join
+      explanation = "This need is not valid because:\n\n#{bulleted_reasons}"
+
       {
         name: "unpublished",
         type: "withdrawal",
         date: need_revision["created_at"],
-        explanation: "Thing."
+        explanation: explanation
       }
     elsif is_valid?(need_revision)
       {