Skip to content

Commit

Permalink
Update scraper.py
Browse files Browse the repository at this point in the history
add env variables to control index, rcount, dcount
  • Loading branch information
Feng-Gao committed Mar 11, 2019
1 parent 9e4138b commit 749908d
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions scraper.py
Expand Up @@ -10,15 +10,15 @@

#taipei provides an api to fetch all metadata
base_url = 'https://data.taipei/opendata/datalist/apiAccess?scope=datasetMetadataSearch&limit=100&offset='
index = 0
index = ENV['MORPH_INDEX']
taipei_url = base_url + str(index)
result = requests.get(taipei_url)
package_count = result.json()['result']['count']
index_count = int(package_count / 100)

dataset_count = 0
resource_count = 0
for i in range(package_count+1):
dataset_count = ENV['MORPH_DCOUNT']
resource_count = ENV['MORPH_RCOUNT']
for i in range(index,package_count+1):
index = i*100
taipei_url = base_url + str(index)
result = requests.get(taipei_url)
Expand Down

0 comments on commit 749908d

Please sign in to comment.