DTL (Test > Levelbuilder): 91ebb935

apps/src/templates/studioHomepages/TeacherHomepage.jsx

@@ -178,7 +178,7 @@ export default class TeacherHomepage extends Component {
         <HeaderBanner headingText={i18n.homepageHeading()} short={true} />
         <ProtectedStatefulDiv ref="flashes" />
         <ProtectedStatefulDiv ref="teacherReminders" />
-        {specialAnnouncement && (
+        {isEnglish && specialAnnouncement && (
           <SpecialAnnouncementActionBlock announcement={specialAnnouncement} />
         )}
         {announcement && showAnnouncement && (

bin/cron/build_contact_rollups_v2

@@ -6,7 +6,8 @@ require 'cdo/only_one'
 
 def main
   contact_rollups = ContactRollupsV2.new
-  contact_rollups.build_and_sync
+  contact_rollups.collect_and_process_contacts
+  contact_rollups.sync_updated_contacts_with_pardot
   contact_rollups.report_results
 end
 

bin/i18n/sync-all.rb

@@ -202,6 +202,13 @@ def create_down_out_pr
     return unless should_i "create the down & out PR"
     `git checkout -B #{DOWN_OUT_BRANCH}`
 
+    I18nScriptUtils.git_add_and_commit(
+      [
+        "bin/i18n/crowdin/*etags.json"
+      ],
+      "etags updates"
+    )
+
     I18nScriptUtils.git_add_and_commit(
       [
         "pegasus/cache",

bin/i18n/sync-down.rb

@@ -5,39 +5,39 @@
 # https://crowdin.com/project/codeorg
 
 require_relative 'i18n_script_utils'
-require 'open3'
+
+require 'cdo/crowdin/utils'
+require 'cdo/crowdin/project'
 
 def sync_down
   I18nScriptUtils.with_synchronous_stdout do
     puts "Beginning sync down"
 
+    logger = Logger.new(STDOUT)
+    logger.level = Logger::INFO
+
     CROWDIN_PROJECTS.each do |name, options|
       puts "Downloading translations from #{name} project"
-      command = "crowdin --config #{options[:config_file]} --identity #{options[:identity_file]} download translations"
-
-      # Filter the output because the crowdin translation download is _super_
-      # verbose; it includes not only a progress spinner, but also information
-      # about each individual file downloaded in each individual language.
-      #
-      # We really only care about general progress monitoring, so we remove or
-      # ignore any things we identify as "noise" in the output.
-      Open3.popen2(command) do |_stdin, stdout, status_thread|
-        while line = stdout.gets
-          # strip out the progress spinner, which is implemented as the sequence
-          # \-/| followed by a backspace character
-          line.gsub!(/[\|\/\-\\][\b]/, '')
-
-          # skip lines detailing individual file extraction
-          next if line.start_with?("Extracting: ")
-
-          # skip warning that happens if the sync is run multiple times in succession
-          next if line == "Warning: Export was skipped. Please note that this method can be invoked only once per 30 minutes.\n"
-
-          puts line
-        end
-
-        raise "Sync down failed"  unless status_thread.value.success?
-      end
+      api_key = YAML.load_file(options[:identity_file])["api_key"]
+      project_id = YAML.load_file(options[:config_file])["project_identifier"]
+      project = Crowdin::Project.new(project_id, api_key)
+      options = {
+        etags_json: File.join(File.dirname(__FILE__), "crowdin", "#{project_id}_etags.json"),
+        locales_dir: File.join(I18N_SOURCE_DIR, '..'),
+        logger: logger
+      }
+      utils = Crowdin::Utils.new(project, options)
+
+      puts "Fetching list of changed files"
+      prefetch = Time.now
+      utils.fetch_changes
+      postfetch = Time.now
+      puts "Changes fetched in #{Time.at(postfetch - prefetch).utc.strftime('%H:%M:%S')}"
+      puts "Downloading changed files"
+      predownload = Time.now
+      utils.download_changed_files
+      postdownload = Time.now
+      puts "Files downloaded in #{Time.at(postdownload - predownload).utc.strftime('%H:%M:%S')}"
     end
 
     puts "Sync down complete"

dashboard/db/schema.rb

@@ -1382,7 +1382,6 @@
     t.boolean  "autoplay_enabled",  default: false,   null: false
     t.index ["code"], name: "index_sections_on_code", unique: true, using: :btree
     t.index ["course_id"], name: "fk_rails_20b1e5de46", using: :btree
-    t.index ["script_id"], name: "fk_rails_5c2401d1cb", using: :btree
     t.index ["user_id"], name: "index_sections_on_user_id", using: :btree
   end
 

dashboard/lib/contact_rollups_v2.rb

@@ -1,7 +1,7 @@
 require 'cdo/log_collector'
 
 class ContactRollupsV2
-  def initialize(is_dry_run: true)
+  def initialize(is_dry_run: false)
     @is_dry_run = is_dry_run
     @log_collector = LogCollector.new('Contact Rollups')
   end

dashboard/test/lib/contact_rollups_v2_test.rb

@@ -28,7 +28,7 @@ class ContactRollupsV2Test < ActiveSupport::TestCase
     PardotV2.stubs(:submit_batch_request).once.returns([])
 
     # Execute the pipeline
-    ContactRollupsV2.new(is_dry_run: false).build_and_sync
+    ContactRollupsV2.new.build_and_sync
 
     # Verify email preference
     pardot_memory_record = ContactRollupsPardotMemory.find_by(email: email_preference.email, pardot_id: 1)
@@ -67,7 +67,7 @@ class ContactRollupsV2Test < ActiveSupport::TestCase
     PardotV2.stubs(:submit_batch_request).once.returns([])
 
     # Execute the pipeline
-    ContactRollupsV2.new(is_dry_run: false).build_and_sync
+    ContactRollupsV2.new.build_and_sync
 
     # Verify results
     pardot_memory_record = ContactRollupsPardotMemory.find_by(email: email, pardot_id: pardot_id)

lib/cdo/crowdin/project.rb

@@ -0,0 +1,113 @@
+require 'httparty'
+
+module Crowdin
+  # This class represents a single project hosted on Crowdin, and provides
+  # access to data on that project via Crowdin's API
+  class Project
+    include HTTParty
+
+    attr_reader :id
+
+    # @param project_id [String]
+    # @param api_key [String]
+    # @see https://crowdin.com/project/codeorg/settings#api for an example of
+    #  how to retrieve these values for the "code.org" project
+    def initialize(project_id, api_key)
+      @id = project_id
+      self.class.base_uri("https://api.crowdin.com/api/project/#{project_id}")
+      self.class.default_params key: api_key
+    end
+
+    # @see https://support.crowdin.com/api/info/
+    def project_info
+      self.class.post("/info")
+    end
+
+    # @param file [String] name of file (within crowdin) to be downloaded
+    # @param language [String] crowdin language code
+    # @param etag [String, nil] the last file version tag returned by crowdin
+    #  for this file. If no changes have occurred since the provided etag was
+    #  generated, crowdin will return a 304 (Not Modified) status instead of
+    #  downloading the file. See the export-file Crowdin documentation for
+    #  details
+    # @param attempts [Number, nil] how many times we should retry the download
+    #  if it fails
+    # @param only_head [Boolean, nil] whether to make a HEAD request rather
+    #  than a full GET request. Defaults to false.
+    # @see https://support.crowdin.com/api/export-file/
+    def export_file(file, language, etag: nil, attempts: 3, only_head: false)
+      options = {
+        query: {
+          file: file,
+          language: language
+        }
+      }
+
+      unless etag.nil?
+        options[:headers] = {
+          "If-None-Match" => etag
+        }
+      end
+
+      only_head ? self.class.head("/export-file", options) : self.class.get("/export-file", options)
+    rescue Net::ReadTimeout => error
+      # Handle a timeout by simply retrying. We default to three attempts before
+      # giving up; if this doesn't work out, other things we could consider:
+      #
+      #   - increasing the default number of attempts
+      #   - increasing the number of attempts for certain high-failure-rate calls
+      #   - increasing the timeout, either globally or for this specific call
+      STDERR.puts "Crowdin.export_file(#{file}) timed out: #{error}"
+      raise if attempts <= 1
+      export_file(file, language, etag: etag, attempts: attempts - 1, only_head: only_head)
+    end
+
+    # Retrieve all languages currently enabled in the crowdin project. Each
+    # language is a hash containing the language name and code, as well as
+    # other internal crowdin values.
+    # @example [{"name"=>"Norwegian", "code"=>"no", "can_translate"=>"1", "can_approve"=>"1"}, ...]
+    # @return [Array<Hash>]
+    def languages
+      project_info["info"]["languages"]["item"]
+    end
+
+    # Retrieve all files currently uploaded to the crowdin project.
+    # @example ["/dashboard/base.yml", "/dashboard/data.yml", ...]
+    # @return [Array<String>]
+    def list_files
+      files = project_info["info"]["files"]["item"]
+      results = []
+      each_file(files) do |file, path|
+        results << File.join(path, file["name"])
+      end
+      results
+    end
+
+    private
+
+    # Iterate through files as returned by crowdin. Crowdin returns files in a
+    # nested format, where each file is a "node", and directories are nodes
+    # that can contain other nodes. This helper simply knows how to traverse
+    # that simulated directory structure, and will yield each file in turn
+    # along with its directory.
+    # @param files [Array<Hash>]
+    # @param path [String, nil]
+    # @yield [name, path] the name of a file and the full path to the directory
+    #   in which it can be found.
+    def each_file(files, path="")
+      files = [files] unless files.is_a? Array
+      files.each do |file|
+        case file["node_type"]
+        when "directory"
+          subfiles = file["files"]["item"]
+          subpath = File.join(path, file["name"])
+          each_file(subfiles, subpath) {|f, p| yield f, p}
+        when "file"
+          yield file, path
+        else
+          raise "Cannot process file of type #{file['node_type']}"
+        end
+      end
+    end
+  end
+end

lib/cdo/crowdin/utils.rb

@@ -0,0 +1,88 @@
+require 'json'
+require 'parallel'
+
+module Crowdin
+  class Utils
+    # @param project [Crowdin::Project]
+    # @param options [Hash, nil]
+    # @param options.changes_json [String, nil] path to file where files with
+    #  changes will be written out in JSON format
+    # @param options.etags_json [String, nil] path to file where etags will be
+    #  written out in JSON format
+    # @param options.locales_dir [String, nil] path to directory where changed
+    #  files should be downloaded
+    # @param options.logger [Logger, nil]
+    def initialize(project, options={})
+      @project = project
+      @changes_json = options.fetch(:changes_json, "/tmp/#{project.id}_changes.json")
+      @etags_json = options.fetch(:etags_json, "/tmp/#{project.id}_etags.json")
+      @locales_dir = options.fetch(:locales_dir, "/tmp/locales")
+      @logger = options.fetch(:logger, Logger.new(STDOUT))
+    end
+
+    # Fetch from Crowdin a list of files changed since the last sync. Uses
+    # etags sourced from the @etags_json file to define what we mean by "since
+    # the last sync," and writes the results out to @changes_json.
+    def fetch_changes
+      etags = File.exist?(@etags_json) ? JSON.parse(File.read(@etags_json)) : {}
+
+      # Clear out existing changes json if it exists
+      File.write(@changes_json, '{}')
+      changes = {}
+
+      languages = @project.languages
+      num_languages = languages.length
+      languages.each_with_index do |language, i|
+        language_code = language["code"]
+        @logger.debug("#{language['name']} (#{language_code}): #{i}/#{num_languages}")
+        @logger.info("~#{(i * 100 / num_languages).round(-1)}% complete (#{i}/#{num_languages})") if i > 0 && i % (num_languages / 5) == 0
+
+        etags[language_code] ||= {}
+        files = @project.list_files
+
+        changed_files = Parallel.map(files) do |file|
+          etag = etags[language_code].fetch(file, nil)
+          response = @project.export_file(file, language_code, etag: etag, only_head: true)
+          case response.code
+          when 200
+            [file, response.headers["etag"]]
+          when 304
+            nil
+          else
+            raise "cannot handle response code #{response.code}"
+          end
+        end.compact
+
+        next if changed_files.empty?
+
+        changes[language_code] = changed_files.to_h
+        etags[language_code].merge!(changes[language_code])
+        File.write(@etags_json, JSON.pretty_generate(etags))
+        File.write(@changes_json, JSON.pretty_generate(changes))
+      end
+    end
+
+    # Downloads all files referenced in @changes_json to @locales_dir
+    def download_changed_files
+      raise "No existing changes json at #{@changes_json}; please run fetch_changes first" unless File.exist?(@changes_json)
+      changes = JSON.parse(File.read(@changes_json))
+      @logger.info("#{changes.keys.length} languages have changes")
+      @project.languages.each do |language|
+        code = language["code"]
+        name = language["name"]
+        files = changes.fetch(code, nil)
+        next unless files.present?
+        filenames = files.keys
+        locale_dir = File.join(@locales_dir, name)
+
+        @logger.debug("#{name} (#{code}): #{filenames.length} files have changes")
+        Parallel.each(filenames) do |file|
+          response = @project.export_file(file, code)
+          dest = File.join(locale_dir, file)
+          FileUtils.mkdir_p(File.dirname(dest))
+          File.write(dest, response.body)
+        end
+      end
+    end
+  end
+end