-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
41 lines (29 loc) · 925 Bytes
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
from GitHubData import GitHubData
from Storage import Storage
from GitHubDataActor import GitHubDataActor
from StorageActor import StorageActor
from queue import Queue
from threading import Thread
import time
import sys
# fetch -> validate(store) -> fetch -> store
project_url = '/repos/Microsoft/vscode'
sock_proxy_host = 'localhost'
sock_proxy_port = 9050
queues = {'fetch':Queue(), 'store':Queue()}
workers = []
for x in range(16):
worker = GitHubDataActor(queues['fetch'], queues, sock_proxy_host, sock_proxy_port)
worker.daemon = True
worker.start()
workers.append(worker)
worker = StorageActor(queues['store'], queues)
worker.daemon = True
worker.start()
workers.append(worker)
queues['fetch'].put(('project_validate', {'project_url': project_url})) #next_queue, link, table_name, extra_data
time.sleep(15)
queues['store'].join()
queues['fetch'].join()
queues['store'].join()
print('Done!')