-
Notifications
You must be signed in to change notification settings - Fork 1
/
deploy.py
132 lines (98 loc) · 2.85 KB
/
deploy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
from fabric.api import *
import os
import re
import json
import pprint
import pdb
import socket
UDP_PORT = 9966
env.hosts = [
"spider@192.168.1.191:22",
"spider@192.168.1.192:22",
"spider@192.168.1.193:22",
"spider@192.168.1.194:22",
]
env.passwords = {
"spider@192.168.1.191:22": "spider#@!",
"spider@192.168.1.192:22": "spider#@!",
"spider@192.168.1.193:22": "spider#@!",
"spider@192.168.1.194:22": "spider#@!",
}
def upload():
with settings(warn_only=True):
run("mkdir ~/work")
os.system("find spider | grep .*pyc. |xargs rm")
os.system("find spider | grep .*sw. |xargs rm")
os.system("tar -cjf spider.tar.bz2 spider")
put("spider.tar.bz2", "~/work/")
run("cd work; rm -rf spider; tar -mxvf spider.tar.bz2")
def install_deps():
_ = lambda x : sudo("apt-get -q -y install %s" % x)
_("pkg-config")
_("python-dev")
_("python-lxml")
_("python-pip")
_("python-mysqldb")
_("libpng12-dev")
_("make cmake")
_("aria2")
_("software-properties-common python-software-properties")
_("autoconf")
_ = lambda x : sudo("apt-get -q -y remove %s" % x)
_("tesseract-ocr")
_("tesseract-ocr-eng")
_("libleptonica")
_("libleptonica-dev")
_("libtesseract3")
_("libtesseract-dev")
__ = lambda x: sudo("pip install %s" %x)
__("redis")
__("hiredis")
__("msgpack-python")
def run_all():
run("cd work; python spider all")
def single_command(command, site=None, worker=False, response=False):
packet = {
"id": 1235,
"cmd": command,
}
if site:
packet["site"] = site
if worker:
packet["worker"] = worker
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind(("0.0.0.0", 9967))
s.sendto(json.dumps(packet), (env.host, UDP_PORT))
if response:
p, addr = s.recvfrom(40960)
s.close()
return json.loads(p)
else:
s.close()
def start_all():
single_command("startall")
def stop_all():
single_command("stopall")
def kill_all():
with settings(warn_only=True):
run("cd ~/work; python spider/kill_worker.py spider")
def stat(site, worker):
resp = single_command("stat", site, worker, response=True)
print "host", env.host
pprint.pprint(resp)
def stop(site, worker):
single_command("stop", site, worker)
def start(site, worker):
single_command("start", site, worker)
def get_all():
resp = single_command("getall", response=True)
print "host", env.host
pprint.pprint(resp)
def refresh():
resp = single_command("refresh", response=True)
print "host", env.host
pprint.pprint(resp)
def setup():
with settings(warn_only=True):
run("cd ~; rm -rf spider")
run("cd ~; mkdir work")