diff --git a/scraper.rb b/scraper.rb index ba414a0..fbcf7f7 100644 --- a/scraper.rb +++ b/scraper.rb @@ -18,8 +18,7 @@ p "Getting first page" first_page = agent.get url # p first_page.body -p "Getting first page again with a js= token" -p first_page.body.scan(/js=-?\d+/)[0] +p "Getting first page again with " + first_page.body.scan(/js=-?\d+/)[0] url_query = url + '?' + first_page.body.scan(/js=-?\d+/)[0] first_page = agent.get url_query @@ -27,8 +26,6 @@ first_page_form = first_page.forms.first # select the "List of Development Applications" radio button first_page_form.radiobuttons[0].click -# next_button = first_page_form.button_with(:value => "Next") -# search_page = agent.submit(first_page_form, next_button, { '__VIEWSTATEENCRYPTED' => '' }) search_page = first_page_form.click_button # select the "Date Lodged" tab @@ -36,32 +33,16 @@ search_form['__EVENTTARGET'] = 'ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$tabControlMenu' search_form['__EVENTARGUMENT'] = '3' search_form['__LASTFOCUS'] = '' -#search_form['__VIEWSTATEENCRYPTED'] = '' -search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mEnquiryListsDropDownList'] = '10' -search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetNameTextBox'] = '' -search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetNumberTextBox'] = '' -search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetTypeDropDown'] = '(any)' -search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mSuburbTextBox'] = '' -search_form['ctl00$mHeight'] = '807' -search_form['ctl00$mWidth'] = '1184' -# search_form['hiddenInputToUpdateATBuffer_CommonToolkitScripts'] = '1' -# date_lodged_link = search_page.link_with(text: 'Date Lodged') +#search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mEnquiryListsDropDownList'] = '10' +#search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetNameTextBox'] = '' +#search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetNumberTextBox'] = '' +#search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mStreetTypeDropDown'] = '(any)' +#search_form['ctl00$MainBodyContent$mGeneralEnquirySearchControl$mTabControl$ctl04$mSuburbTextBox'] = '' +#search_form['ctl00$mHeight'] = '807' +#search_form['ctl00$mWidth'] = '1184' p "Clicking Date Lodged tab" - search_page = agent.submit(search_form) -#search_page = agent.submit(search_form, nil, { -#'Host' => 'online.unley.sa.gov.au', -#'Connection' => 'keep-alive', -#'Cache-Control' => 'max-age=0', -#'Origin' => 'https://online.unley.sa.gov.au', -#'Upgrade-Insecure-Requests' => '1', -#'Content-Type' => 'application/x-www-form-urlencoded', -#'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36', -#'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', -#'Referer' => 'https://online.unley.sa.gov.au/ePathway/Production/Web/GeneralEnquiry/EnquirySearch.aspx', -#'Accept-Encoding' => 'gzip, deflate, br', -#'Accept-Language' => 'en-US,en;q=0.9' -#}) + p "Searching" # p search_page.title.strip