Merge remote-tracking branch 'documentcloud/master' into dc-master

app/controllers/admin_controller.rb

@@ -375,6 +375,13 @@ def login_as
     acc = Account.lookup(params[:email])
     return not_found unless acc
     acc.authenticate(session, cookies)
+
+    # notify when an Admin logs in as a user.
+    hook_url = DC::SECRETS['slack_webhook']
+    text = "#{@current_account.full_name} has logged in as #{acc.full_name} (#{acc.email} on #{Rails.env})."
+    data = {:payload => {:text => text, :username => "docbot", :icon_emoji => ":doccloud:"}.to_json}
+    RestClient.post(hook_url, data)
+
     redirect_to '/'
   end
 

app/controllers/download_controller.rb

@@ -55,6 +55,7 @@ def send_viewer
         @current_document = doc
         @local = true
         @embed_options = {container: '#viewer'}
+        @asset_root = "."
         html = ERB.new(File.read("#{Rails.root}/app/views/documents/show.html.erb")).result(binding)
         html.gsub!(/\="\/viewer\//, '="viewer/')
         zip.get_output_stream("#{doc.slug}.html") {|f| f.write(html) }

app/controllers/home_controller.rb

@@ -1,3 +1,6 @@
+require 'addressable/uri'
+require 'openssl'
+
 class HomeController < ApplicationController
   include DC::Access
 
@@ -47,6 +50,10 @@ def api_terms
     @canonical_url = api_terms_url if @version == @current_terms['version']
     render layout: 'new', template: "home/api_terms/show"
   end
+
+  def status
+    not_found
+  end
 
   def privacy
     render layout: 'new'
@@ -55,6 +62,44 @@ def privacy
   def faq
     render layout: 'new'
   end
+
+  def quackbot
+    if current_account
+      user_data = {email: current_account.email, slug: current_organization.slug}
+      # user_data = {email: 'ted@knowtheory.net', slug: 'biffud'}
+      cipher = OpenSSL::Cipher::AES.new(256, :CBC)
+      cipher.encrypt
+
+      key = Base64.decode64(DC::SECRETS["quackbot_cipher_key"])
+      iv  = cipher.random_iv
+
+      cipher.key = key
+      cipher.iv = iv
+
+      encrypted = cipher.update(user_data.to_json) + cipher.final
+
+      state_data = "#{Base64.encode64(encrypted)}--#{Base64.encode64(iv).chomp}"
+
+      #msg, ivStr = state_data.split("--").map{|str| Base64.decode64(str)}
+      #
+      #decipher = OpenSSL::Cipher::AES.new(256, :CBC)
+      #decipher.decrypt
+      #decipher.key = key
+      #decipher.iv = ivStr
+      #
+      #plain = decipher.update(msg) + decipher.final
+
+      @slack_url = Addressable::URI.parse("https://slack.com/oauth/authorize")
+      @slack_url.query_values = {
+        client_id: "2309601577.242920041892",
+        scope: "bot,chat:write:bot,emoji:read,files:read,links:write,im:read,im:write,incoming-webhook,commands",
+        state: state_data
+      }
+
+    end
+
+    render layout: 'new'
+  end
 
   private
 

app/controllers/workspace_controller.rb

@@ -43,5 +43,4 @@ def help
   def upgrade
     render :layout => nil
   end
-
 end

app/models/document.rb

@@ -877,6 +877,9 @@ def queue_import(opts={})
     ).queue
   end
 
+  # you probably want to call this as
+  # Document.duplicate(ids, account, 
+  #  {projects: project_ids, include_sections: true, include_docdata: true, include_annotations: true})
   def self.duplicate(document_ids, account, options={})
     RestClient.post(DC::CONFIG['cloud_crowd_server'] + '/jobs', {:job => {
       'action'  => 'duplicate_documents',
@@ -888,6 +891,11 @@ def self.duplicate(document_ids, account, options={})
   end
 
   # Create an identical clone of this document, in all ways (except for the ID).
+  # toggles to include various aspects of documents:
+  # include_docdata => true/false
+  # include_sections => true/false
+  # include_annotations => true/false & conditioned on if the recipient should be treated as owner
+  # include_project => true/false: copy the documents into the same projects the original doc is in.
   def duplicate!(recipient=nil, options={})
     Document.transaction do
       # Clone the document.
@@ -904,9 +912,8 @@ def duplicate!(recipient=nil, options={})
 
       # Clone the docdata.
       if docdata and options['include_docdata']
-        Docdata.create! docdata.attributes.merge newattrs
+        Docdata.create! document_id: copy_id, data: docdata.data
       end
-
 
       # Clone the associations.
       associations = [entities, entity_dates, pages]

app/views/documents/_iframe_embed_code.html.erb

@@ -3,7 +3,7 @@
     <%= content_tag :iframe, nil, {
           src: options[:document].iframe_embed_src_url(data),
           title: "#{options[:document].title} (Hosted by DocumentCloud)",
-          sandbox: ['allow-scripts', 'allow-same-origin', 'allow-popups'],
+          sandbox: ['allow-scripts', 'allow-same-origin', 'allow-popups', 'allow-forms'],
           frameborder: '0',
           style: 'position:absolute;top:0;left:0;width:100%;height:100%;border:1px solid #aaa;border-bottom:0;box-sizing:border-box;' } %>
   </div>

app/views/documents/show.html.erb

@@ -38,13 +38,13 @@
   %>
   <%= render partial: 'common/sharing_meta_tags' if @sharing_meta_tags.present? %>
   <!--[if (!IE)|(gte IE 8)]><!-->
-    <link href="<%= DC.asset_root(agnostic: true) %>/viewer/viewer-datauri.css" media="screen" rel="stylesheet" type="text/css" />
+    <link href="<%= @asset_root || DC.asset_root(agnostic: true) %>/viewer/viewer-datauri.css" media="screen" rel="stylesheet" type="text/css" />
   <!--<![endif]-->
   <!--[if lte IE 7]>
     <link href="<%= DC.asset_root(agnostic: true) %>/viewer/viewer.css" media="screen" rel="stylesheet" type="text/css" />
   <![endif]-->
-  <link href="<%= DC.asset_root(agnostic: true) %>/viewer/printviewer.css" media="print" rel="stylesheet" type="text/css" />
-  <script src="<%= DC.asset_root(agnostic: true) %>/viewer/viewer.js"></script>
+  <link href="<%= @asset_root || DC.asset_root(agnostic: true) %>/viewer/printviewer.css" media="print" rel="stylesheet" type="text/css" />
+  <script src="<%= @asset_root || DC.asset_root(agnostic: true) %>/viewer/viewer.js"></script>
   <% if Rails.env.production? %>
     <script>
       window.DV.recordHit = "<%= DC.server_root(agnostic: true).html_safe %>/pixel.gif";

app/views/home/contact.html.erb

@@ -35,7 +35,7 @@
 
   <h2>General Questions</h2>
 
-  <p>For non-technical questions or general help, please email <a href="mailto:info@documentcloud.org?subject=question">info@documentcloud.org</a> or phone (202) 505-1010.</p>
+  <p>For non-technical questions or general help, please email <a href="mailto:info@documentcloud.org?subject=question">info@documentcloud.org</a>.</p>
 
 
 

app/views/home/p3p.html

@@ -27,7 +27,6 @@ <h2>About Us</h2>
 <p>You may contact us by e-mail at 
 <a href="mailto:support@documentcloud.org">
 support@documentcloud.org</a>. 
-You may call us at (202) 505-1010.
 <!-- "Privacy Seals" section of privacy policy -->
 <h2>Dispute Resolution and Privacy Seals</h2>
 <p>We have no privacy seals or other assurance mechanisms.

app/views/home/p3p.xml

@@ -12,7 +12,6 @@
     <!-- Description of the entity making this policy statement. -->
     <ENTITY>
     <DATA-GROUP>
-<DATA ref="#business.contact-info.telecom.telephone.number">(202) 505-1010</DATA>
 <DATA ref="#business.contact-info.online.email">support@documentcloud.org</DATA>
 <DATA ref="#business.contact-info.online.uri">https://www.documentcloud.org/</DATA>
 <DATA ref="#business.contact-info.postal.organization">DocumentCloud</DATA>

app/views/home/quackbot.html.erb

@@ -0,0 +1,25 @@
+<% @title = 'Hi Quackbot' %>
+<div class="content paper container-md">
+  <h1>Say Hi to Quackbot</h1>
+  <section>
+    <% if @current_account %>
+    <p>Hey <%= @current_account.full_name %>, let us introduce you and <b><%= @current_organization.name %></b> to <span style="background:#eef7fd;color:#0576b9;">@quackbot</span>!</p>
+    <p>
+      <%= image_tag('/images/home/quackbot-icon_128.png', size: '128x128', alt: 'quackbot logo', class: 'img-thumbnail') %>
+    </p>
+    <p>
+    <a href="<%= @slack_url %>"><img alt="Add to Slack" height="40" width="139" src="https://platform.slack-edge.com/img/add_to_slack.png" srcset="https://platform.slack-edge.com/img/add_to_slack.png 1x, https://platform.slack-edge.com/img/add_to_slack@2x.png 2x" /></a>
+    </p>
+    <p>We built <span style="background:#eef7fd;color:#0576b9;">@quackbot</span> to help journalists like you!  Ask <span style="background:#eef7fd;color:#0576b9;">@quackbot</span> how it can help you for more details.</p>
+    <p><span style="background:#eef7fd;color:#0576b9;">@quackbot</span> is young and we're still teaching it new things, so if <span style="background:#eef7fd;color:#0576b9;">@quackbot</span> does anything weird (or you encounter any bugs), <%= mail_to "support@documentcloud.org", "please let us know!" %></p>
+    <% else %>
+    <p>
+      <%= image_tag('/images/home/quackbot-icon_128.png', size: '128x128', alt: 'quackbot logo', class: 'img-thumbnail') %>
+    </p>
+    <p>Want to add Quackbot to your slack?</p>
+    <p>Get started by <%= link_to "logging into DocumentCloud", login_path(next: quackbot_path) %></p>
+
+    <% end %>
+  </section>
+</div>
+

app/views/home/terms/changelog.yml

@@ -5,7 +5,8 @@
 - version:     2.0
   description: 'See <a href="https://blog.documentcloud.org/blog/2016/04/updating-documentcloud-terms-of-service/">blog post</a> for details on changes.'
   start:       '2016-05-25'
+  end:         '2017-07-31'
 - version:     2.1
   current:     true
-  description: 'DocumentCloud moves to its own independent non-profit'
+  description: 'DocumentCloud moves to its own independent non-profit.'
   start:       '2017-08-01'

app/views/layouts/workspace.html.erb

@@ -22,7 +22,6 @@
   <%= csrf_meta_tag %> 
 </head>
 <body class="<%= @bodyclass %> <%= @current_account ? 'logged_in' : '' %>">
-  <%= render :partial => 'common/maintenance' %>
   <div id="container">
     <div id="topbar" class="header gradient_mid">
       <div id="logo">

config/document_cloud.yml

@@ -21,15 +21,15 @@ development:
 
 staging:
   <<: *defaults
-  cloud_crowd_server:   <%= secrets['db_host'] %>:8080
+  cloud_crowd_server:   <%= secrets['central_server'] %>
   cloud_crowd_console:  staging.documentcloud.org:8080
   server_root:          staging.documentcloud.org
   asset_root:           s3.amazonaws.com/dcloud_staging
   workspace_asset_root: s3.amazonaws.com/dcloud_staging
 
 production:
   <<: *defaults
-  cloud_crowd_server:   <%= secrets['db_host'] %>:8080
+  cloud_crowd_server:   <%= secrets['central_server'] %>
   cloud_crowd_console:  db01.documentcloud.org:8080
   server_root:          www.documentcloud.org
   asset_root:           assets.documentcloud.org

config/locales/eng.yml

@@ -319,8 +319,6 @@
       released”'
   - force_ocr: Force OCR
   - form_contact_instructions: Use this form (or email to %s) to contact us for assistance.
-      If you need to speak to someone immediately, you can call us at (202) 505-1010.
-      See %s for more ways to get in touch.
   - freelancer: Freelancer
   - full_page: Full Page
   - future_documents_w_appear: Future documents %s will appear in your embed.

config/routes.rb

@@ -176,9 +176,11 @@
   get '/opensource',            to: 'home#opensource',    as: 'opensource'
   get '/about',                 to: 'home#about',         as: 'about'
   get '/contact',               to: 'home#contact',       as: 'contact'
+  get '/status',                to: 'home#status',        as: 'status'
   get '/help',                  to: 'home#help'
   get '/help/:page',            to: 'home#help'
   get '/multilanguage',         to: 'home#multilanguage', as: 'multilanguage'
+  get '/quackbot',              to: 'home#quackbot',      as: 'quackbot'
 
   # Redirects
   get '/index.php',             to: 'redirect#index',                     url: '/'

config/server/scripts/create-ami-commands.sh

@@ -33,6 +33,7 @@ rails runner -e production "puts Organization.count" # check for human eyeballs
 sudo mkdir /mnt/cloud_crowd
 sudo chown ubuntu:ubuntu /mnt/cloud_crowd
 
+# Set up the Rails Log directory to point to it's own device (so it doesn't chew up the main file system)
 sudo mkdir /mnt/log
 sudo mount /dev/xvdc /mnt/log
 sudo chown ubuntu:ubuntu /mnt/log

config/sunspot.yml

@@ -1,13 +1,13 @@
 production:
   solr:
-    hostname: ip-10-69-172-218.ec2.internal
+    hostname: ip-10-0-0-16.ec2.internal
     port: 7377
     path: /solr/collection1
     read_timeout: 5
     open_timeout: 1.5
     log_level: INFO
     min_memory: 5G
-    max_memory: 7G
+    max_memory: 6G
     pid_dir: tmp/pids
 
 development:

lib/tasks/cron.rake

@@ -3,7 +3,7 @@ namespace :cron do
   # Only run on db01
   task :nightly do
     invoke 'crowd:cluster:free_calais_blacklist'
-    invoke 'db:backup'
+    invoke 'db:backup' unless you_are_documentcloud?
     invoke 'db:vacuum_analyze'
     invoke 'mail:csv'
   end
@@ -25,4 +25,8 @@ private
 
 def invoke(name)
   Rake::Task[name].invoke
-end
+end
+
+def you_are_documentcloud?
+  File.exists? File.join(Rails.root, 'secrets', 'documentcloud.yep')
+end