Skip to content

Commit

Permalink
Add workflows
Browse files Browse the repository at this point in the history
  • Loading branch information
@gabifija
gabifija committed Sep 28, 2020
1 parent dff7dc5 commit 11582a1
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 111 deletions.
25 changes: 19 additions & 6 deletions lib/filestack/config.rb
View file
Expand Up @@ -7,11 +7,6 @@ class FilestackConfig
CDN_URL = 'https://cdn.filestackcontent.com'.freeze
PROCESS_URL = 'https://process.filestackapi.com'.freeze

MULTIPART_START_URL = 'https://upload.filestackapi.com/multipart/start'.freeze
MULTIPART_UPLOAD_URL = 'https://upload.filestackapi.com/multipart/upload'.freeze
MULTIPART_COMMIT_URL = 'https://upload.filestackapi.com/multipart/commit'.freeze
MULTIPART_COMPLETE_URL = 'https://upload.filestackapi.com/multipart/complete'.freeze

MULTIPART_PARAMS = %w[
store_location store_region store_container
store_path store_access
Expand All @@ -22,10 +17,28 @@ class FilestackConfig
VERSION = Filestack::Ruby::VERSION
HEADERS = {
'User-Agent' => "filestack-ruby #{VERSION}",
'Filestack-Source' => "Ruby-#{VERSION}"
'Filestack-Source' => "Ruby-#{VERSION}",
'Content-Type' => "application/json",
'Accept-Encoding' => "application/json"
}.freeze

INTELLIGENT_ERROR_MESSAGES = ['BACKEND_SERVER', 'BACKEND_NETWORK', 'S3_SERVER', 'S3_NETWORK']

def self.multipart_start_url
"https://upload.filestackapi.com/multipart/start"
end

def self.multipart_upload_url(base_url)
"https://#{base_url}/multipart/upload"
end

def self.multipart_commit_url(base_url)
"https://#{base_url}/multipart/commit"
end

def self.multipart_complete_url(base_url)
"https://#{base_url}/multipart/complete"
end
end

class TransformConfig
Expand Down
28 changes: 11 additions & 17 deletions lib/filestack/models/filestack_client.rb
View file
Expand Up @@ -18,34 +18,28 @@ class FilestackClient
# @param [String] apikey Your Filestack API key
# @param [FilestackSecurity] security A Filestack security object,
# if security is enabled
def initialize(apikey, security: nil)
def initialize(apikey, storage: 'S3', security: nil)
@apikey = apikey
@security = security
@storage = storage
end

# Upload a local file or external url
# @param [String] filepath The path of a local file
# @param [String] external_url An external URL
# @param [Bool] multipart Switch for miltipart
# (Default: true)
# @param [Hash] options User-supplied upload options
#
# return [Filestack::FilestackFilelink]
def upload(filepath: nil, external_url: nil, multipart: true, options: {}, storage: 's3', intelligent: false, timeout: 60)
if filepath && external_url
return 'You cannot upload a URL and file at the same time'
end
response = if filepath && multipart
multipart_upload(@apikey, filepath, @security, options, timeout, storage, intelligent: intelligent)
def upload(filepath: nil, external_url: nil, options: {}, intelligent: false, timeout: 60)
return 'You cannot upload a URL and file at the same time' if filepath && external_url

response = if filepath
multipart_upload(@apikey, filepath, @security, options, timeout, @storage, intelligent: intelligent)
else
send_upload(
@apikey,
filepath: filepath,
external_url: external_url,
options: options,
security: @security,
storage: storage
)
send_upload(@apikey,
external_url: external_url,
options: options,
security: @security)
end
FilestackFilelink.new(response['handle'], security: @security, apikey: @apikey)
end
Expand Down
80 changes: 31 additions & 49 deletions lib/filestack/utils/multipart_upload_utils.rb
View file
Expand Up @@ -23,16 +23,6 @@ def get_file_info(file)
[filename, filesize, mimetype.to_s]
end

def multipart_options(options)
[:region, :container, :path, :access].each do |key|
if options.has_key?(key)
options[:"store_#{key}"] = options[key]
options.delete(key)
end
end
return options
end

# Send start response to multipart endpoint
#
# @param [String] apikey Filestack API key
Expand All @@ -51,23 +41,21 @@ def multipart_start(apikey, filename, filesize, mimetype, security, storage, opt
filename: filename,
mimetype: mimetype,
size: filesize,
store_location: storage,
file: Tempfile.new(filename),
multipart: intelligent
store: { location: storage },
fii: intelligent
}

options = multipart_options(options)
params = params.merge!(options) if options
params[:store].merge!(options) if options

unless security.nil?
params[:policy] = security.policy
params[:signature] = security.signature
end

response = Typhoeus.post(
FilestackConfig::MULTIPART_START_URL, body: params,
headers: FilestackConfig::HEADERS
)
response = Typhoeus.post(FilestackConfig.multipart_start_url,
body: params.to_json,
headers: FilestackConfig::HEADERS)

if response.code == 200
JSON.parse(response.body)
else
Expand All @@ -93,7 +81,7 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
seek_point = 0
while seek_point < filesize
part_info = {
seek: seek_point,
seek_point: seek_point,
filepath: filepath,
filename: filename,
apikey: apikey,
Expand All @@ -104,10 +92,10 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
upload_id: start_response['upload_id'],
location_url: start_response['location_url'],
start_response: start_response,
store_location: storage
store: { location: storage }
}
options = multipart_options(options)
part_info = part_info.merge!(options) if options

part_info[:store].merge!(options) if options

if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
size = filesize - (seek_point)
Expand Down Expand Up @@ -135,9 +123,9 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
# multipart uploads
#
# @return [Typhoeus::Response]
def upload_chunk(job, apikey, filepath, options)
def upload_chunk(job, apikey, filepath, options, storage)
file = File.open(filepath)
file.seek(job[:seek])
file.seek(job[:seek_point])
chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)

md5 = Digest::MD5.new
Expand All @@ -150,14 +138,14 @@ def upload_chunk(job, apikey, filepath, options)
uri: job[:uri],
region: job[:region],
upload_id: job[:upload_id],
store_location: job[:store_location],
store: { location: storage },
file: Tempfile.new(job[:filename])
}
data = data.merge!(options) if options
fs_response = Typhoeus.post(
FilestackConfig::MULTIPART_UPLOAD_URL, body: data,
headers: FilestackConfig::HEADERS
).body

fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
body: data.to_json,
headers: FilestackConfig::HEADERS).body
fs_response = JSON.parse(fs_response)
Typhoeus.put(
fs_response['url'], headers: fs_response['headers'], body: chunk
Expand All @@ -172,17 +160,17 @@ def upload_chunk(job, apikey, filepath, options)
# multipart uploads
#
# @return [Array] Array of parts/etags strings
def run_uploads(jobs, apikey, filepath, options)
def run_uploads(jobs, apikey, filepath, options, storage)
bar = ProgressBar.new(jobs.length)
results = Parallel.map(jobs, in_threads: 4) do |job|
response = upload_chunk(
job, apikey, filepath, options
job, apikey, filepath, options, storage
)
if response.code == 200
bar.increment!
part = job[:part]
etag = response.headers[:etag]
"#{part}:#{etag}"
{ part_number: part, etag: etag }
end
end
results
Expand Down Expand Up @@ -213,17 +201,14 @@ def multipart_complete(apikey, filename, filesize, mimetype, start_response, par
filename: filename,
size: filesize,
mimetype: mimetype,
store_location: storage,
file: Tempfile.new(filename)
store: { location: storage },
}
options = multipart_options(options)
data.merge!(options) if options
data.merge!(intelligent ? { multipart: intelligent } : { parts: parts_and_etags.join(';') })
data[:store].merge!(options) if options
data.merge!(intelligent ? { fii: intelligent } : { parts: parts_and_etags })

Typhoeus.post(
FilestackConfig::MULTIPART_COMPLETE_URL, body: data,
headers: FilestackConfig::HEADERS
)
Typhoeus.post(FilestackConfig.multipart_complete_url(start_response['location_url']),
body: data.to_json,
headers: FilestackConfig::HEADERS)
end

# Run entire multipart process through with file and options
Expand All @@ -238,27 +223,24 @@ def multipart_complete(apikey, filename, filesize, mimetype, start_response, par
# @return [Hash]
def multipart_upload(apikey, filepath, security, options, timeout, storage, intelligent: false)
filename, filesize, mimetype = get_file_info(filepath)

start_response = multipart_start(
apikey, filename, filesize, mimetype, security, storage, options, intelligent
)

unless start_response['upload_type'].nil?
intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
end

jobs = create_upload_jobs(
apikey, filename, filepath, filesize, start_response, storage, options
)

if intelligent_enabled
if intelligent
state = IntelligentState.new
run_intelligent_upload_flow(jobs, state)
run_intelligent_upload_flow(jobs, state, storage)
response_complete = multipart_complete(
apikey, filename, filesize, mimetype,
start_response, nil, options, storage, intelligent
)
else
parts_and_etags = run_uploads(jobs, apikey, filepath, options)
parts_and_etags = run_uploads(jobs, apikey, filepath, options, storage)
response_complete = multipart_complete(
apikey, filename, filesize, mimetype,
start_response, parts_and_etags, options, storage
Expand Down

0 comments on commit 11582a1

Please sign in to comment.