-
Notifications
You must be signed in to change notification settings - Fork 445
/
bartercast_crawler_plugin.py
155 lines (128 loc) · 6.01 KB
/
bartercast_crawler_plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
"""
Run Dispersy in standalone bartercast crawler mode.
Crawls BarterCommunity and collects interaction data from peers.
"""
import os
import signal
from Tribler.dispersy.crypto import NoVerifyCrypto, NoCrypto
# from dispersy.discovery.community import DiscoveryCommunity
from Tribler.dispersy.dispersy import Dispersy
from Tribler.dispersy.endpoint import StandaloneEndpoint
from Tribler.dispersy.exception import CommunityNotFoundException
from twisted.application.service import IServiceMaker, MultiService
from twisted.conch import manhole_tap
from twisted.internet import reactor
from twisted.plugin import IPlugin
from twisted.python import usage
from twisted.python.log import msg
from twisted.python.threadable import isInIOThread
from zope.interface import implements
from Tribler.community.bartercast4.community import BarterCommunityCrawler
class BartercastCrawler(Dispersy):
def __init__(self, endpoint, working_directory, silent=False, crypto=NoVerifyCrypto()):
super(BartercastCrawler, self).__init__(endpoint, working_directory, u":memory:", crypto)
# location of persistent storage
self._persistent_storage_filename = os.path.join(working_directory, "persistent-storage.data")
self._silent = silent
self._my_member = None
def start(self):
assert isInIOThread()
if super(BartercastCrawler, self).start():
self._create_my_member()
print "loading bartercc as member %s: " % self._my_member
# self.register_task("unload inactive communities",
# LoopingCall(self.unload_inactive_communities)).start(COMMUNITY_CLEANUP_INTERVAL)
self.define_auto_load(BarterCommunityCrawler, self._my_member, (), load=True)
# self.define_auto_load(TrackerHardKilledCommunity, self._my_member)
# if not self._silent:
# self._statistics_looping_call = LoopingCall(self._report_statistics)
# self._statistics_looping_call.start(300)
return True
return False
def _create_my_member(self):
# generate a new my-member
ec = self.crypto.generate_key(u"very-low")
self._my_member = self.get_member(private_key=self.crypto.key_to_bin(ec))
@property
def persistent_storage_filename(self):
return self._persistent_storage_filename
def get_community(self, cid, load=False, auto_load=True):
try:
return super(BartercastCrawler, self).get_community(cid, True, True)
except CommunityNotFoundException:
return BarterCommunityCrawler.init_community(self, self.get_member(mid=cid), self._my_member)
class Options(usage.Options):
optFlags = [
["profiler" , "P", "use cProfile on the Dispersy thread"],
["memory-dump", "d", "use meliae to dump the memory periodically"],
["silent" , "s", "Prevent tracker printing to console"],
]
optParameters = [
["statedir", "s", "." , "Use an alternate statedir" , str],
["ip" , "i", "0.0.0.0" , "Dispersy uses this ip" , str],
["port" , "p", 6421 , "Dispersy uses this UDL port" , int],
["crypto" , "c", "ECCrypto", "The Crypto object type Dispersy is going to use" , str],
["manhole" , "m", 0 , "Enable manhole telnet service listening at the specified port", int],
]
class BartercastCrawlerServiceMaker(object):
implements(IServiceMaker, IPlugin)
tapname = "bartercast_crawler"
description = "A BartercastCommunity statistics crawler"
options = Options
def makeService(self, options):
"""
Construct a dispersy service.
"""
tracker_service = MultiService()
tracker_service.setName("Bartercast Crawler")
# crypto
if options["crypto"] == 'NoCrypto':
crypto = NoCrypto()
else:
crypto = NoVerifyCrypto()
container = [None]
manhole_namespace = {}
if options["manhole"]:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace': manhole_namespace,
'telnetPort': 'tcp:%d:interface=127.0.0.1' % port,
'sshPort': None,
'passwd': os.path.join(os.path.dirname(__file__), 'passwd'),
})
tracker_service.addService(manhole)
manhole.startService()
def run():
# setup
dispersy = BartercastCrawler(StandaloneEndpoint(options["port"],
options["ip"]),
unicode(options["statedir"]),
bool(options["silent"]),
crypto)
container[0] = dispersy
manhole_namespace['dispersy'] = dispersy
self._stopping = False
def signal_handler(sig, frame):
msg("Received signal '%s' in %s (shutting down)" % (sig, frame))
if not self._stopping:
self._stopping = True
try:
dispersy.stop()
except Exception, e:
msg("Got exception when stopping dispersy: %s" % e)
reactor.stop()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
# start
print "starting dispersy"
if not dispersy.start():
raise RuntimeError("Unable to start Dispersy")
# wait forever
reactor.exitCode = 0
reactor.callWhenRunning(run)
# TODO: exit code
return tracker_service
# Now construct an object which *provides* the relevant interfaces
# The name of this variable is irrelevant, as long as there is *some*
# name bound to a provider of IPlugin and IServiceMaker.
serviceMaker = BartercastCrawlerServiceMaker()