multipart_upload_utils.rb

require 'base64'
require 'timeout'
require 'digest'
require 'mimemagic'
require 'json'
require 'parallel'
require 'typhoeus'
require 'progress_bar'
require 'filestack/config'
require 'filestack/utils/utils'

include UploadUtils
include IntelligentUtils
# Includes all the utility functions for Filestack multipart uploads
module MultipartUploadUtils
  def get_file_info(file)
    filename = File.basename(file)
    filesize = File.size(file)
    mimetype = MimeMagic.by_magic(File.open(file))
    if mimetype.nil?
      mimetype = 'application/octet-stream'
    end
    [filename, filesize, mimetype.to_s]
  end

  # Send start response to multipart endpoint
  #
  # @param [String]             apikey        Filestack API key
  # @param [String]             filename      Name of incoming file
  # @param [Int]                filesize      Size of incoming file
  # @param [String]             mimetype      Mimetype of incoming file
  # @param [FilestackSecurity]  security      Security object with
  #                                           policy/signature
  # @param [Hash]               options       User-defined options for
  #                                           multipart uploads
  #
  # @return [Typhoeus::Response]
  def multipart_start(apikey, filename, filesize, mimetype, security, storage, options = {}, intelligent)
    params = {
      apikey: apikey,
      filename: filename,
      mimetype: mimetype,
      size: filesize,
      store: { location: storage },
      fii: intelligent
    }

    params[:store].merge!(options) if options

    unless security.nil?
      params[:policy] = security.policy
      params[:signature] = security.signature
    end

    response = Typhoeus.post(FilestackConfig.multipart_start_url,
                             body: params.to_json,
                             headers: FilestackConfig::HEADERS)

    if response.code == 200
      JSON.parse(response.body)
    else
      raise RuntimeError.new(response.body)
    end
  end

  # Create array of jobs for parallel uploading
  #
  # @param [String]             apikey         Filestack API key
  # @param [String]             filename       Name of incoming file
  # @param [String]             filepath       Local path to file
  # @param [Int]                filesize       Size of incoming file
  # @param [Typhoeus::Response]  start_response Response body from
  #                                            multipart_start
  # @param [Hash]               options        User-defined options for
  #                                            multipart uploads
  #
  # @return [Array]
  def create_upload_jobs(apikey, filename, filepath, filesize, start_response, storage, options)
    jobs = []
    part = 1
    seek_point = 0
    while seek_point < filesize
      part_info = {
        seek_point: seek_point,
        filepath: filepath,
        filename: filename,
        apikey: apikey,
        part: part,
        filesize: filesize,
        uri: start_response['uri'],
        region: start_response['region'],
        upload_id: start_response['upload_id'],
        location_url: start_response['location_url'],
        start_response: start_response,
        store: { location: storage }
      }

      part_info[:store].merge!(options) if options

      if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
        size = filesize - (seek_point)
      else
        size = FilestackConfig::DEFAULT_CHUNK_SIZE
      end
      part_info[:size] = size
      jobs.push(part_info)
      part += 1
      seek_point += FilestackConfig::DEFAULT_CHUNK_SIZE
    end
    jobs
  end


  # Uploads one chunk of the file
  #
  # @param [Hash]               job            Hash of options needed
  #                                            to upload a chunk
  # @param [String]             apikey         Filestack API key
  # @param [String]             location_url   Location url given back
  #                                            from endpoint
  # @param [String]             filepath       Local path to file
  # @param [Hash]               options        User-defined options for
  #                                            multipart uploads
  #
  # @return [Typhoeus::Response]
  def upload_chunk(job, apikey, filepath, options, storage)
    file = File.open(filepath)
    file.seek(job[:seek_point])
    chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)

    md5 = Digest::MD5.new
    md5 << chunk
    data = {
      apikey: apikey,
      part: job[:part],
      size: chunk.length,
      md5: md5.base64digest,
      uri: job[:uri],
      region: job[:region],
      upload_id: job[:upload_id],
      store: { location: storage },
      file: Tempfile.new(job[:filename])
    }
    data = data.merge!(options) if options

    fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
                                body: data.to_json,
                                headers: FilestackConfig::HEADERS).body
    fs_response = JSON.parse(fs_response)
    Typhoeus.put(
      fs_response['url'], headers: fs_response['headers'], body: chunk
    )
  end
  # Runs all jobs in parallel
  #
  # @param [Array]              jobs           Array of jobs to be run
  # @param [String]             apikey         Filestack API key
  # @param [String]             filepath       Local path to file
  # @param [Hash]               options        User-defined options for
  #                                            multipart uploads
  #
  # @return [Array]                            Array of parts/etags strings
  def run_uploads(jobs, apikey, filepath, options, storage)
    bar = ProgressBar.new(jobs.length)
    results = Parallel.map(jobs, in_threads: 4) do |job|
      response = upload_chunk(
        job, apikey, filepath, options, storage
      )
      if response.code == 200
        bar.increment!
        part = job[:part]
        etag = response.headers[:etag]
        { part_number: part, etag: etag }
      end
    end
    results
  end
  # Send complete call to multipart endpoint
  #
  # @param [String]             apikey          Filestack API key
  # @param [String]             filename        Name of incoming file
  # @param [Int]                filesize        Size of incoming file
  # @param [String]             mimetype        Mimetype of incoming file
  # @param [Typhoeus::Response]  start_response  Response body from
  #                                             multipart_start
  # @param [FilestackSecurity]  security        Security object with
  #                                             policy/signature
  # @param [Array]              parts_and_etags Array of strings defining
  #                                             etags and their associated
  #                                             part numbers
  # @param [Hash]               options         User-defined options for
  #                                             multipart uploads
  #
  # @return [Typhoeus::Response]
  def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, storage, intelligent = false)
    data = {
      apikey: apikey,
      uri: start_response['uri'],
      region: start_response['region'],
      upload_id: start_response['upload_id'],
      filename: filename,
      size: filesize,
      mimetype: mimetype,
      store: { location: storage },
    }
    data[:store].merge!(options) if options
    data.merge!(intelligent ? { fii: intelligent } : { parts: parts_and_etags })

    Typhoeus.post(FilestackConfig.multipart_complete_url(start_response['location_url']),
                  body: data.to_json,
                  headers: FilestackConfig::HEADERS)
  end

  # Run entire multipart process through with file and options
  #
  # @param [String]             apikey          Filestack API key
  # @param [String]             filename        Name of incoming file
  # @param [FilestackSecurity]  security        Security object with
  #                                             policy/signature
  # @param [Hash]               options         User-defined options for
  #                                             multipart uploads
  #
  # @return [Hash]
  def multipart_upload(apikey, filepath, security, options, timeout, storage, intelligent: false)
    filename, filesize, mimetype = get_file_info(filepath)

    start_response = multipart_start(
      apikey, filename, filesize, mimetype, security, storage, options, intelligent
    )

    jobs = create_upload_jobs(
      apikey, filename, filepath, filesize, start_response, storage, options
    )

    if intelligent
      state = IntelligentState.new
      run_intelligent_upload_flow(jobs, state, storage)
      response_complete = multipart_complete(
        apikey, filename, filesize, mimetype,
        start_response, nil, options, storage, intelligent
      )
    else
      parts_and_etags = run_uploads(jobs, apikey, filepath, options, storage)
      response_complete = multipart_complete(
        apikey, filename, filesize, mimetype,
        start_response, parts_and_etags, options, storage
      )
    end
    begin
      Timeout::timeout(timeout) {
        while response_complete.code == 202
          response_complete = multipart_complete(
            apikey, filename, filesize, mimetype,
            start_response, nil, options, storage, intelligent
          )
        end
      }
    rescue
      raise "Upload timed out upon completion. Please try again later"
    end
    JSON.parse(response_complete.body)
  end
end