diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7bd0d69 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/ +scraperwiki.sqlite diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..7d1f4cf --- /dev/null +++ b/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' +ruby '2.4.1' + +gem 'rest-client' +gem 'json' +gem 'scraperwiki', git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..9cfbbf6 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,46 @@ +GIT + remote: https://github.com/openaustralia/scraperwiki-ruby.git + revision: fc50176812505e463077d5c673d504a6a234aa78 + branch: morph_defaults + specs: + scraperwiki (3.0.1) + httpclient + sqlite_magic + +GEM + remote: https://rubygems.org/ + specs: + domain_name (0.5.20170404) + unf (>= 0.0.5, < 1.0.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + json (2.1.0) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + netrc (0.11.0) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + sqlite3 (1.3.13) + sqlite_magic (0.0.6) + sqlite3 + unf (0.1.4) + unf_ext + unf_ext (0.0.7.4) + +PLATFORMS + ruby + +DEPENDENCIES + json + rest-client + scraperwiki! + +RUBY VERSION + ruby 2.4.1p111 + +BUNDLED WITH + 1.15.4 diff --git a/README.md b/README.md index 8a44f1a..ea67d25 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# atak \ No newline at end of file +# digitalne mesto + +simple scrapper of [www.digitalnemesto.sk] \ No newline at end of file diff --git a/scraper.rb b/scraper.rb new file mode 100644 index 0000000..a2c8107 --- /dev/null +++ b/scraper.rb @@ -0,0 +1,37 @@ +require 'scraperwiki' +require 'rest-client' +require 'json' + +# API +# https://www.digitalnemesto.sk/DmApi/ + +API_URL = 'https://www.digitalnemesto.sk/DmApi/json/reply'.freeze +DOC_TYPES = [{ key: 'FakturyDodavatelske', unique_id: 'FakturaId', sqlite_tr: 'SumaFaktSDph' }, + { key: 'FakturyOdberatelske', unique_id: 'FakturaId', sqlite_tr: 'SumaFaktSDph' }, + { key: 'zmluvy', unique_id: 'ZmluvaId', sqlite_tr: 'SumaSdph' }, + { key: 'objednavkyDosle', unique_id: 'ObjednavkaId', sqlite_tr: 'SumaSdph' }].freeze +ORG_URL = 'https://www.digitalnemesto.sk/DmApi/json/reply/Organizacie'.freeze + +def documents(org) + DOC_TYPES.each do |doc| + p "--> #{org['mesto']} : #{doc[:key]}" + items = JSON.parse(RestClient.post("#{API_URL}/#{doc[:key]}", OrganizaciaId: org['id'], Year: 0)) + items['Data'].each do |item| + # sqlite true || false + item[doc[:sqlite_tr]] | item[doc[:sqlite_tr]] = item[doc[:sqlite_tr]].to_s + ScraperWiki.save([doc[:unique_id]], item.merge(org), table_name = doc[:key]) + end + end +end + +def orgs + orgs = JSON.parse(RestClient.get(ORG_URL)) + orgs['Data'].each do |org| + ScraperWiki.save(['OrganizaciaId'], org, table_name = 'Organizacie') + documents('id' => org['OrganizaciaId'], 'mesto' => org['NazovObec']) + end +end + +p Time.now +orgs +p Time.now \ No newline at end of file