Skip to content

Commit

Permalink
Merge d73d86a into dff7dc5
Browse files Browse the repository at this point in the history
  • Loading branch information
@gabifija
gabifija committed Sep 28, 2020
2 parents dff7dc5 + d73d86a commit 97b07fe
Show file tree
Hide file tree
Showing 6 changed files with 181 additions and 160 deletions.
29 changes: 19 additions & 10 deletions README.md
View file
Expand Up @@ -44,41 +44,50 @@ Or install it yourself as:
require 'filestack'
```
Intialize the client using your API key, and security if you are using it.

```ruby
client = FilestackClient.new('YOUR_API_KEY', security: security_object)
```
### Uploading
Filestack uses multipart uploading by default, which is faster for larger files. This can be turned off by passing in ```multipart: false```. Multipart is disabled when uploading external URLs.
```ruby
filelink = client.upload(filepath: '/path/to/file')

filelink = client.upload(filepath: '/path/to/file', multipart: false)
filelink = client.upload(filepath: '/path/to/localfile')

# OR

filelink = client.upload(external_url: 'http://someurl.com')
filelink = client.upload(external_url: 'http://domain.com/image.png')
```

To upload a local and an external file with query parameters:
```ruby
filelink = client.upload(filepath: '/path/to/file', options: {mimetype: 'image/png'})
filelink = client.upload(filepath: '/path/to/localfile', options: { mimetype: 'image/png' })

filelink = client.upload(external_url: 'http://someurl.com/image.png', options: {mimetype: 'image/jpeg'})
filelink = client.upload(external_url: 'http://domain.com/image.png', options: { mimetype: 'image/jpeg' })
```

To store file on `dropbox`, `azure`, `gcs` or `rackspace`, you must have the chosen provider configured in the developer portal to enable this feature. By default the file is stored on `s3`. You can add more details of the storage in `options`.

```ruby
filelink = client.upload(filepath: '/path/to/file', storage: 'dropbox', options: {path: 'folder_name/'})
filelink = client.upload(filepath: '/path/to/file', storage: 's3', options: { path: 'folder_name/', container: 'container_name', location: 's3', region: 'region_name' })

filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { location: 'dropbox', path: 'folder_name' })
```

### Workflows
Workflows allow you to wire up conditional logic and image processing to enforce business processes, automate ingest, and save valuable development time. In order to trigger the workflow job for each upload:

```ruby
filelink = client.upload(filepath: '/path/to/file', options: { workflows: ["workflow_id_1", "workflow_id_2"] })

#OR

filelink = client.upload(external_url: 'http://someurl.com/image.png', storage: 'dropbox', options: {path: 'folder_name/'})
filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { workflows: ["workflow_id_1"] })
```

### Security
If security is enabled on your account, or if you are using certain actions that require security (delete, overwrite and certain transformations), you will need to create a security object and pass it into the client on instantiation.

```ruby
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick]})
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick runWorkflow]})
client = FilestackClient.new('YOUR_API_KEY', security: security)
```

Expand Down
25 changes: 19 additions & 6 deletions lib/filestack/config.rb
View file
Expand Up @@ -7,11 +7,6 @@ class FilestackConfig
CDN_URL = 'https://cdn.filestackcontent.com'.freeze
PROCESS_URL = 'https://process.filestackapi.com'.freeze

MULTIPART_START_URL = 'https://upload.filestackapi.com/multipart/start'.freeze
MULTIPART_UPLOAD_URL = 'https://upload.filestackapi.com/multipart/upload'.freeze
MULTIPART_COMMIT_URL = 'https://upload.filestackapi.com/multipart/commit'.freeze
MULTIPART_COMPLETE_URL = 'https://upload.filestackapi.com/multipart/complete'.freeze

MULTIPART_PARAMS = %w[
store_location store_region store_container
store_path store_access
Expand All @@ -22,10 +17,28 @@ class FilestackConfig
VERSION = Filestack::Ruby::VERSION
HEADERS = {
'User-Agent' => "filestack-ruby #{VERSION}",
'Filestack-Source' => "Ruby-#{VERSION}"
'Filestack-Source' => "Ruby-#{VERSION}",
'Content-Type' => "application/json",
'Accept-Encoding' => "application/json"
}.freeze

INTELLIGENT_ERROR_MESSAGES = ['BACKEND_SERVER', 'BACKEND_NETWORK', 'S3_SERVER', 'S3_NETWORK']

def self.multipart_start_url
"https://upload.filestackapi.com/multipart/start"
end

def self.multipart_upload_url(base_url)
"https://#{base_url}/multipart/upload"
end

def self.multipart_commit_url(base_url)
"https://#{base_url}/multipart/commit"
end

def self.multipart_complete_url(base_url)
"https://#{base_url}/multipart/complete"
end
end

class TransformConfig
Expand Down
23 changes: 8 additions & 15 deletions lib/filestack/models/filestack_client.rb
View file
Expand Up @@ -26,26 +26,19 @@ def initialize(apikey, security: nil)
# Upload a local file or external url
# @param [String] filepath The path of a local file
# @param [String] external_url An external URL
# @param [Bool] multipart Switch for miltipart
# (Default: true)
# @param [Hash] options User-supplied upload options
#
# return [Filestack::FilestackFilelink]
def upload(filepath: nil, external_url: nil, multipart: true, options: {}, storage: 's3', intelligent: false, timeout: 60)
if filepath && external_url
return 'You cannot upload a URL and file at the same time'
end
response = if filepath && multipart
def upload(filepath: nil, external_url: nil, options: {}, intelligent: false, timeout: 60, storage: 'S3')
return 'You cannot upload a URL and file at the same time' if filepath && external_url

response = if filepath
multipart_upload(@apikey, filepath, @security, options, timeout, storage, intelligent: intelligent)
else
send_upload(
@apikey,
filepath: filepath,
external_url: external_url,
options: options,
security: @security,
storage: storage
)
send_upload(@apikey,
external_url: external_url,
options: options,
security: @security)
end
FilestackFilelink.new(response['handle'], security: @security, apikey: @apikey)
end
Expand Down
80 changes: 31 additions & 49 deletions lib/filestack/utils/multipart_upload_utils.rb
View file
Expand Up @@ -23,16 +23,6 @@ def get_file_info(file)
[filename, filesize, mimetype.to_s]
end

def multipart_options(options)
[:region, :container, :path, :access].each do |key|
if options.has_key?(key)
options[:"store_#{key}"] = options[key]
options.delete(key)
end
end
return options
end

# Send start response to multipart endpoint
#
# @param [String] apikey Filestack API key
Expand All @@ -51,23 +41,21 @@ def multipart_start(apikey, filename, filesize, mimetype, security, storage, opt
filename: filename,
mimetype: mimetype,
size: filesize,
store_location: storage,
file: Tempfile.new(filename),
multipart: intelligent
store: { location: storage },
fii: intelligent
}

options = multipart_options(options)
params = params.merge!(options) if options
params[:store].merge!(options) if options

unless security.nil?
params[:policy] = security.policy
params[:signature] = security.signature
end

response = Typhoeus.post(
FilestackConfig::MULTIPART_START_URL, body: params,
headers: FilestackConfig::HEADERS
)
response = Typhoeus.post(FilestackConfig.multipart_start_url,
body: params.to_json,
headers: FilestackConfig::HEADERS)

if response.code == 200
JSON.parse(response.body)
else
Expand All @@ -93,7 +81,7 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
seek_point = 0
while seek_point < filesize
part_info = {
seek: seek_point,
seek_point: seek_point,
filepath: filepath,
filename: filename,
apikey: apikey,
Expand All @@ -104,10 +92,10 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
upload_id: start_response['upload_id'],
location_url: start_response['location_url'],
start_response: start_response,
store_location: storage
store: { location: storage }
}
options = multipart_options(options)
part_info = part_info.merge!(options) if options

part_info[:store].merge!(options) if options

if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
size = filesize - (seek_point)
Expand Down Expand Up @@ -135,9 +123,9 @@ def create_upload_jobs(apikey, filename, filepath, filesize, start_response, sto
# multipart uploads
#
# @return [Typhoeus::Response]
def upload_chunk(job, apikey, filepath, options)
def upload_chunk(job, apikey, filepath, options, storage)
file = File.open(filepath)
file.seek(job[:seek])
file.seek(job[:seek_point])
chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)

md5 = Digest::MD5.new
Expand All @@ -150,14 +138,14 @@ def upload_chunk(job, apikey, filepath, options)
uri: job[:uri],
region: job[:region],
upload_id: job[:upload_id],
store_location: job[:store_location],
store: { location: storage },
file: Tempfile.new(job[:filename])
}
data = data.merge!(options) if options
fs_response = Typhoeus.post(
FilestackConfig::MULTIPART_UPLOAD_URL, body: data,
headers: FilestackConfig::HEADERS
).body

fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
body: data.to_json,
headers: FilestackConfig::HEADERS).body
fs_response = JSON.parse(fs_response)
Typhoeus.put(
fs_response['url'], headers: fs_response['headers'], body: chunk
Expand All @@ -172,17 +160,17 @@ def upload_chunk(job, apikey, filepath, options)
# multipart uploads
#
# @return [Array] Array of parts/etags strings
def run_uploads(jobs, apikey, filepath, options)
def run_uploads(jobs, apikey, filepath, options, storage)
bar = ProgressBar.new(jobs.length)
results = Parallel.map(jobs, in_threads: 4) do |job|
response = upload_chunk(
job, apikey, filepath, options
job, apikey, filepath, options, storage
)
if response.code == 200
bar.increment!
part = job[:part]
etag = response.headers[:etag]
"#{part}:#{etag}"
{ part_number: part, etag: etag }
end
end
results
Expand Down Expand Up @@ -213,17 +201,14 @@ def multipart_complete(apikey, filename, filesize, mimetype, start_response, par
filename: filename,
size: filesize,
mimetype: mimetype,
store_location: storage,
file: Tempfile.new(filename)
store: { location: storage },
}
options = multipart_options(options)
data.merge!(options) if options
data.merge!(intelligent ? { multipart: intelligent } : { parts: parts_and_etags.join(';') })
data[:store].merge!(options) if options
data.merge!(intelligent ? { fii: intelligent } : { parts: parts_and_etags })

Typhoeus.post(
FilestackConfig::MULTIPART_COMPLETE_URL, body: data,
headers: FilestackConfig::HEADERS
)
Typhoeus.post(FilestackConfig.multipart_complete_url(start_response['location_url']),
body: data.to_json,
headers: FilestackConfig::HEADERS)
end

# Run entire multipart process through with file and options
Expand All @@ -238,27 +223,24 @@ def multipart_complete(apikey, filename, filesize, mimetype, start_response, par
# @return [Hash]
def multipart_upload(apikey, filepath, security, options, timeout, storage, intelligent: false)
filename, filesize, mimetype = get_file_info(filepath)

start_response = multipart_start(
apikey, filename, filesize, mimetype, security, storage, options, intelligent
)

unless start_response['upload_type'].nil?
intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
end

jobs = create_upload_jobs(
apikey, filename, filepath, filesize, start_response, storage, options
)

if intelligent_enabled
if intelligent
state = IntelligentState.new
run_intelligent_upload_flow(jobs, state)
run_intelligent_upload_flow(jobs, state, storage)
response_complete = multipart_complete(
apikey, filename, filesize, mimetype,
start_response, nil, options, storage, intelligent
)
else
parts_and_etags = run_uploads(jobs, apikey, filepath, options)
parts_and_etags = run_uploads(jobs, apikey, filepath, options, storage)
response_complete = multipart_complete(
apikey, filename, filesize, mimetype,
start_response, parts_and_etags, options, storage
Expand Down

0 comments on commit 97b07fe

Please sign in to comment.