-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.py
70 lines (54 loc) · 3.6 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
import requests
from pprint import pprint
import json
from multiprocessing.dummy import Pool as ThreadPool
import time
def read_config(filename):
data = None
with open(filename, 'r') as f:
data = json.loads(f.read())
return data
config = read_config('config.json')
puzzle_ids = []
game_url_template = "https://nyt-games-prd.appspot.com/svc/crosswords/v6/game/"
security_header = config['s-header']
user_id = config['user-id']
game_data = []
# for multithreading
urls = [
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-01-01&date_end=2019-02-28',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-03-01&date_end=2019-04-30',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-05-01&date_end=2019-06-30',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-07-01&date_end=2019-07-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-08-01&date_end=2019-10-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2019-11-01&date_end=2020-01-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2020-02-01&date_end=2020-03-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2020-04-01&date_end=2020-06-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2020-07-01&date_end=2020-08-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2020-09-01&date_end=2020-11-31',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2020-12-01&date_end=2021-02-28',
'https://nyt-games-prd.appspot.com/svc/crosswords/v3/' + user_id + '/puzzles.json?publish_type=mini&sort_order=asc&sort_by=print_date&date_start=2021-03-01&date_end=2021-03-08'
]
def get_cword_data(cword):
data_url = game_url_template + str(cword['puzzle_id']) + ".json"
cword_response = requests.get(data_url, headers=security_header)
cword_response_data = json.loads(cword_response.content)
cword["board"] = cword_response_data["board"]
return cword
def main():
start = time.time()
# scrape crossword lists for publish dates and game ids
for url in urls:
response = requests.get(url, headers=security_header)
data = json.loads(response.content)['results']
solved_cwords = [cword for cword in data if cword['solved']]
with ThreadPool(20) as pool:
results = pool.map(get_cword_data, solved_cwords)
game_data.extend(results)
# output the list of boards somehow (for now, writing to a json file)
with open("data.json", "w") as f:
f.write(json.dumps(game_data))
end = time.time()
print("script finished in", end - start, "s")
if __name__ == "__main__":
main()