Skip to content

Commit

Permalink
Merge pull request #45 from nrmay/master
Browse files Browse the repository at this point in the history 
Refactor Eventbrite ingestor
  • Loading branch information
@dresa-org-au
dresa-org-au committed Apr 20, 2022
2 parents 3dd0efe + 23ff4a8 commit 5f54342
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 124 deletions.
241 changes: 118 additions & 123 deletions lib/ingestors/ingestor_event_rest.rb
View file
Expand Up @@ -21,7 +21,6 @@ def initialize
end

def read(url)
@messages << "#{self.class.name}.#{__method__} url[#{url}] token[#{@token}]"
begin
process = nil

Expand Down Expand Up @@ -50,13 +49,12 @@ def read(url)

def process_eventbrite(url)
records_read = 0
records_draft = 0
records_inactive = 0
records_expired = 0
records_completed = 0

begin
# initialise next_page
next_page = "#{url}/events/?token=#{@token}"
next_page = "#{url}/events/?status=live&token=#{@token}"

while next_page
# execute REST request
Expand All @@ -81,104 +79,95 @@ def process_eventbrite(url)
unless events.nil? or events.empty?
events.each do |item|
records_read += 1
unless item['status'].nil?
# check status
case item['status']
when 'draft'
records_draft += 1
when 'completed'
records_completed += 1
when 'live'
# create new event
event = Event.new

# check for expired
event.timezone = item['start']['timezone']
event.start = item['start']['local']
event.end = item['end']['local']
if event.expired?
records_expired += 1
if item['status'].nil? or item['status'] != 'live'
records_inactive += 1
else
# create new event
event = Event.new

# check for expired
event.timezone = item['start']['timezone']
event.start = item['start']['local']
event.end = item['end']['local']
if event.expired?
records_expired += 1
else
# set required attributes
event.title = item['name']['text'] unless item['name'].nil?
event.url = item['url']
event.description = convert_description item['description']['html'] unless item['description'].nil?
if item['online_event'].nil? or item['online_event'] == false
event.online = false
else
# set required attributes
event.title = item['name']['text'] unless item['name'].nil?
event.url = item['url']
event.description = convert_description item['description']['html'] unless item['description'].nil?
if item['online_event'].nil? or item['online_event'] == false
event.online = false
else
event.online = true
end

# organizer
organizer = get_eventbrite_organizer item['organizer_id']
event.organizer = organizer['name'] unless organizer.nil?

# address fields
venue = get_eventbrite_venue item['venue_id']
unless venue.nil? or venue['address'].nil?
address = venue['address']
venue = address['address_1']
venue += (', ' + address['address_2']) unless address['address_2'].blank?
event.venue = venue
event.city = address['city']
event.country = address['country']
event.postcode = address['postal_code']
event.latitude = address['latitude']
event.longitude = address['longitude']
end

# set optional attributes
event.keywords = []
category = get_eventbrite_category item['category_id']
subcategory = get_eventbrite_subcategory(
item['subcategory_id'], item['category_id'])
event.keywords << category['name'] unless category.nil?
event.keywords << subcategory['name'] unless subcategory.nil?

unless item['capacity'].nil? or item['capacity'] == 'null'
event.capacity = item['capacity'].to_i
end

event.event_types = []
format = get_eventbrite_format item['format_id']
unless format.nil?
type = convert_event_types format['short_name']
event.event_types << type unless type.nil?
end

if item['invite_only'].nil? or !item['invite_only']
event.eligibility = 'open_to_all'
else
event.eligibility = 'by_invitation'
end

if item['is_free'].nil? or !item['is_free']
event.cost_basis = 'charge'
event.cost_currency = item['currency']
else
event.cost_basis = 'free'
end

# add event to events array
add_event(event)
@ingested += 1
event.online = true
end
else
@messages << "Extract event failed with unhandled status: #{item['status']}"

# organizer
organizer = get_eventbrite_organizer item['organizer_id']
event.organizer = organizer['name'] unless organizer.nil?

# address fields
venue = get_eventbrite_venue item['venue_id']
unless venue.nil? or venue['address'].nil?
address = venue['address']
venue = address['address_1']
venue += (', ' + address['address_2']) unless address['address_2'].blank?
event.venue = venue
event.city = address['city']
event.country = address['country']
event.postcode = address['postal_code']
event.latitude = address['latitude']
event.longitude = address['longitude']
end

# set optional attributes
event.keywords = []
category = get_eventbrite_category item['category_id']
subcategory = get_eventbrite_subcategory(
item['subcategory_id'], item['category_id'])
event.keywords << category['name'] unless category.nil?
event.keywords << subcategory['name'] unless subcategory.nil?

unless item['capacity'].nil? or item['capacity'] == 'null'
event.capacity = item['capacity'].to_i
end

event.event_types = []
format = get_eventbrite_format item['format_id']
unless format.nil?
type = convert_event_types format['short_name']
event.event_types << type unless type.nil?
end

if item['invite_only'].nil? or !item['invite_only']
event.eligibility = 'open_to_all'
else
event.eligibility = 'by_invitation'
end

if item['is_free'].nil? or !item['is_free']
event.cost_basis = 'charge'
event.cost_currency = item['currency']
else
event.cost_basis = 'free'
end

# add event to events array
add_event(event)
@ingested += 1
end
end
rescue Exception => e
@messages << "Extract event fields failed with: #{e.message}"
end
end
end

@messages << "Eventbrite events read[#{records_read}] draft[#{records_draft}] expired[#{records_expired}] completed[#{records_completed}]"
rescue Exception => e
@messages << "#{self.class} failed with: #{e.message}"
end

# finished
@messages << "Eventbrite events ingestor: records read[#{records_read}] inactive[#{records_inactive}] expired[#{records_expired}]"
return
end

Expand Down Expand Up @@ -287,67 +276,73 @@ def populate_eventbrite_categories
end

def get_eventbrite_subcategory(id, category_id)
category = get_eventbrite_category category_id

# abort on bad input
return nil if category.nil? or id.nil? or id == 'null'
return nil if id.nil? or id == 'null'

# get subcategories
# get category
category = get_eventbrite_category category_id
return nil if category.nil?

# get subcategories from cache
subcategories = category['subcategories']

if subcategories.nil?
# populate subcategories
begin
url = "#{category['resource_uri']}?token=#{@token}"
response = get_JSON_response url
unless response.nil?
# updated cached category
@eventbrite_objects[:categories][id] = response
subcategories = response['subcategories']
end
rescue Exception => e
@messages << "get Eventbrite subcategory failed with: #{e.message}"
end
end
# get subcategories from query
subcategories = populate_eventbrite_subcategories id, category if subcategories.nil?

# check for subcategory
unless subcategories.nil? and !subcategories.kind_of?(Array)
if !subcategories.nil? and subcategories.kind_of?(Array)
subcategories.each { |sub| return sub if sub['id'] == id }
end

# not found
nil
end

def populate_eventbrite_subcategories(id, category)
subcategories = nil
begin
url = "#{category['resource_uri']}?token=#{@token}"
response = get_JSON_response url
# updated cached category
@eventbrite_objects[:categories][id] = response
subcategories = response['subcategories']
rescue Exception => e
@messages << "get Eventbrite subcategory failed with: #{e.message}"
end
return subcategories
end

def get_eventbrite_organizer(id)
# fields: description (text, html), long_description (text, html),
# resource_uri, id, name, url, etc.
# abort on bad input
return nil if id.nil? or id == 'null'

# initialize cache
@eventbrite_objects[:organizers] = {} if @eventbrite_objects[:organizers].nil?

# abort on bad input
return nil if id.nil? or id == 'null'

# not in cache
unless @eventbrite_objects[:organizers].keys.include? id
begin
# get from query and add to cache if found
url = "https://www.eventbriteapi.com/v3/organizers/#{id}/?token=#{@token}"
organizer = get_JSON_response url
@eventbrite_objects[:organizers][id] = organizer unless organizer.nil?
rescue Exception => e
@messages << "get Eventbrite Venue failed with: #{e.message}"
end
populate_eventbrite_organizer id
end

# return from cache
@eventbrite_objects[:organizers][id]
end

def populate_eventbrite_organizer(id)
begin
# get from query and add to cache if found
url = "https://www.eventbriteapi.com/v3/organizers/#{id}/?token=#{@token}"
organizer = get_JSON_response url
# add to cache
@eventbrite_objects[:organizers][id] = organizer unless organizer.nil?
rescue Exception => e
@messages << "get Eventbrite Venue failed with: #{e.message}"
end
end

def process_elixir(url)
# execute REST request
results = get_JSON_response url
results = get_JSON_response url, 'application/vnd.api+json'
data = results['data']

# extract materials from results
Expand Down Expand Up @@ -406,11 +401,11 @@ def process_elixir(url)
end
end

def get_JSON_response(url)
def get_JSON_response(url, accept_params = 'application/json')
response = RestClient::Request.new(method: :get,
url: CGI.unescape_html(url),
verify_ssl: false,
headers: { accept: 'application/vnd.api+json' }).execute
headers: { accept: accept_params }).execute
# check response
raise "invalid response code: #{response.code}" unless response.code == 200
JSON.parse(response.to_str)
Expand Down
2 changes: 1 addition & 1 deletion test/tasks/rake_task_event_rest.rb
View file
Expand Up @@ -216,7 +216,7 @@ class RakeTaskEventRest < ActiveSupport::TestCase
assert_respond_to source, :log
#noinspection RubyNilAnalysis
refute_nil source.log
message = 'Eventbrite events read[68] draft[2] expired[0] completed[53]'
message = 'Eventbrite events ingestor: records read[18] inactive[5] expired[0]'
assert_includes source.log, message

# check logfile messages
Expand Down
2 changes: 2 additions & 0 deletions test/test_helper.rb
View file
Expand Up @@ -178,6 +178,8 @@ def mock_ingestions
to_return(status: 200, headers: {}, body: eventbrite_ardc_body)
WebMock.stub_request(:get,'https://www.eventbriteapi.com/v3/organizations/34338661734/events/?page=2&token=YXAKB2UNBVO7FV5SJHQA').
to_return(status: 200, headers: {}, body: eventbrite_ardc_2_body)
WebMock.stub_request(:get,'https://www.eventbriteapi.com/v3/organizations/34338661734/events/?status=live&token=YXAKB2UNBVO7FV5SJHQA').
to_return(status: 200, headers: {}, body: eventbrite_ardc_2_body)
WebMock.stub_request(:get, 'https://www.eventbriteapi.com/v3/organizers/14317910674/?token=YXAKB2UNBVO7FV5SJHQA').
to_return(status: 200, headers: {}, body: eventbrite_organizer_14317910674 )
WebMock.stub_request(:get, 'https://www.eventbriteapi.com/v3/organizers/8082048069/?token=YXAKB2UNBVO7FV5SJHQA').
Expand Down

0 comments on commit 5f54342

Please sign in to comment.