Skip to content
Browse files

Allows searching for members, squares, and text (incompatible DB stru…

…cture, remove old databases)
  • Loading branch information...
1 parent 34479a3 commit 107e241bc95058ac9c07f1298663a9444ee5b1b4 @eviy eviy committed Apr 30, 2012
Showing with 407 additions and 79 deletions.
  1. +9 −3 src/chat-demo.py
  2. +177 −20 src/discovery/community.py
  3. +112 −20 src/discovery/conversion.py
  4. +68 −16 src/discovery/payload.py
  5. +30 −10 src/square/community.py
  6. +11 −10 src/square/database.py
View
12 src/chat-demo.py
@@ -108,14 +108,20 @@ def DEBUG_SIMULATION(self):
def response_func(message):
if message:
- dprint("received ", len(message.payload.hots), " hot messages")
+ if message.name == "search-member-response":
+ dprint("received ", len(message.payload.members), " members")
+ if message.name == "search-square-response":
+ dprint("received ", len(message.payload.squares), " squares")
+ if message.name == "search-text-response":
+ dprint("received ", len(message.payload.texts), " texts")
else:
dprint("received timeout, will occur for each search unless 999 responses are received")
- self._discovery.keyword_search([u"test",], response_func)
for index in xrange(999999):
# user clicked the 'search' button
- self._discovery.keyword_search([u"SIM", u"%d" % index], response_func)
+ self._discovery.simple_member_search(u"member test %d" % index, response_func)
+ self._discovery.simple_square_search(u"square test %d" % index, response_func)
+ self._discovery.simple_text_search(u"text test %d" % index, response_func)
yield 1.0
def onTextMessageReceived(self, message):
View
197 src/discovery/community.py
@@ -1,11 +1,14 @@
+from itertools import islice
+from os import path, makedirs
from random import sample
from time import time
from conversion import Conversion
-from payload import HotsPayload, SearchPayload, SearchResponsePayload
+from payload import HotsPayload, SearchMemberRequestPayload, SearchSquareRequestPayload, SearchTextRequestPayload, SearchMemberResponsePayload, SearchSquareResponsePayload, SearchTextResponsePayload
from hot import Hot, HotCache
from square.community import PreviewCommunity
+from square.database import SquareDatabase
from dispersy.member import DummyMember
from dispersy.cache import CacheDict
@@ -23,6 +26,17 @@
class DiscoveryCommunity(Community):
def __init__(self, *args):
super(DiscoveryCommunity, self).__init__(*args)
+
+ self._database = SquareDatabase.has_instance()
+ if not self._database:
+ # our data storage
+ sqlite_directory = path.join(self._dispersy.working_directory, u"sqlite")
+ if not path.isdir(sqlite_directory):
+ makedirs(sqlite_directory)
+
+ self._database = SquareDatabase.get_instance(sqlite_directory)
+ self._dispersy.database.attach_commit_callback(self._database.commit)
+
self._explicitly_hot_text = []
self._implicitly_hot_text = []
self._top_squares = []
@@ -34,8 +48,12 @@ def __init__(self, *args):
def initiate_meta_messages(self):
return [Message(self, u"hots", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=5), HotsPayload(), self._dispersy._generic_timeline_check, self.on_hots),
- Message(self, u"search", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=20), SearchPayload(), self._dispersy._generic_timeline_check, self.on_search),
- Message(self, u"search-response", NoAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), SearchResponsePayload(), self._dispersy._generic_timeline_check, self.on_search_response)]
+ Message(self, u"search-member-request", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=20), SearchMemberRequestPayload(), self._dispersy._generic_timeline_check, self.on_search_member_request),
+ Message(self, u"search-square-request", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=20), SearchSquareRequestPayload(), self._dispersy._generic_timeline_check, self.on_search_square_request),
+ Message(self, u"search-text-request", NoAuthentication(), PublicResolution(), DirectDistribution(), CommunityDestination(node_count=20), SearchTextRequestPayload(), self._dispersy._generic_timeline_check, self.on_search_text_request),
+ Message(self, u"search-member-response", NoAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), SearchMemberResponsePayload(), self._dispersy._generic_timeline_check, self.on_search_member_response),
+ Message(self, u"search-square-response", NoAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), SearchSquareResponsePayload(), self._dispersy._generic_timeline_check, self.on_search_square_response),
+ Message(self, u"search-text-response", NoAuthentication(), PublicResolution(), DirectDistribution(), CandidateDestination(), SearchTextResponsePayload(), self._dispersy._generic_timeline_check, self.on_search_text_response)]
def initiate_conversions(self):
return [DefaultConversion(self), Conversion(self)]
@@ -139,33 +157,172 @@ def on_hots(self, messages):
if len(self._top_squares) + len(self._top_text) < 10:
self._collect_top_hots()
- def keyword_search(self, keywords, response_func, response_args=(), timeout=10.0):
- return self.expression_search(u"|".join(keywords), response_func, response_args, timeout)
+ def simple_member_search(self, string, response_func, response_args=(), timeout=10.0):
+ pairs = [(min(len(keyword), 127), keyword) for keyword in string.split()]
+ return self.member_search(pairs, [], 1, response_func, response_args, timeout)
+
+ def simple_square_search(self, string, response_func, response_args=(), timeout=10.0):
+ pairs = [(min(len(keyword), 127), keyword) for keyword in string.split()]
+ return self.square_search(pairs, [], 1, response_func, response_args, timeout)
+
+ def simple_text_search(self, string, response_func, response_args=(), timeout=10.0):
+ pairs = [(min(len(keyword), 127), keyword) for keyword in string.split()]
+ return self.text_search(pairs, [], 1, response_func, response_args, timeout)
+
+ def member_search(self, terms, squares, threshold, response_func, response_args=(), timeout=10.0):
+ meta = self._meta_messages[u"search-member-request"]
+ message = meta.impl(distribution=(self.global_time,), payload=(0, terms, squares, threshold))
+ if self._dispersy.store_update_forward([message], False, False, True):
+ meta = self._meta_messages[u"search-member-response"]
+ self._dispersy.await_message(meta.generate_footprint(), response_func, response_args=response_args, timeout=timeout, max_responses=1)
+ else:
+ if __debug__: dprint("unable to search. most likely there are no candidates", level="warning")
+ response_func(None, *response_args)
+
+ return message
+
+ def square_search(self, terms, squares, threshold, response_func, response_args=(), timeout=10.0):
+ meta = self._meta_messages[u"search-square-request"]
+ message = meta.impl(distribution=(self.global_time,), payload=(0, terms, squares, threshold))
+ if self._dispersy.store_update_forward([message], False, False, True):
+ meta = self._meta_messages[u"search-square-response"]
+ self._dispersy.await_message(meta.generate_footprint(), response_func, response_args=response_args, timeout=timeout, max_responses=1)
+ else:
+ if __debug__: dprint("unable to search. most likely there are no candidates", level="warning")
+ response_func(None, *response_args)
+
+ return message
- def expression_search(self, expression, response_func, response_args=(), timeout=10.0):
- meta = self._meta_messages[u"search"]
- message = meta.impl(distribution=(self.global_time,), payload=(expression,))
+ def text_search(self, terms, squares, threshold, response_func, response_args=(), timeout=10.0):
+ meta = self._meta_messages[u"search-text-request"]
+ message = meta.impl(distribution=(self.global_time,), payload=(0, terms, squares, threshold))
if self._dispersy.store_update_forward([message], False, False, True):
- meta = self._meta_messages[u"search-response"]
- self._dispersy.await_message(meta.generate_footprint(), response_func, response_args=response_args, timeout=timeout, max_responses=999)
+ meta = self._meta_messages[u"search-text-response"]
+ self._dispersy.await_message(meta.generate_footprint(), response_func, response_args=response_args, timeout=timeout, max_responses=1)
else:
if __debug__: dprint("unable to search. most likely there are no candidates", level="warning")
response_func(None, *response_args)
return message
- def on_search(self, messages):
- meta = self._meta_messages[u"search-response"]
+ def on_search_member_request(self, messages):
+ dispersy_execute = self._dispersy.database.execute
+ execute = self._database.execute
+ meta = self._meta_messages[u"search-member-response"]
+ responses = []
+
for message in messages:
- if __debug__: dprint("searching \\", message.payload.expression, "\\ for ", message.candidate)
+ payload = message.payload
+ if __debug__: dprint("searching ", " + ".join("%d:%s" % (weight, term) for weight, term in payload.terms), " for ", message.candidate)
+
+ results = dict()
+ for weight, term in payload.terms:
+ for docid, in execute(u"SELECT docid FROM member_fts WHERE alias MATCH ?", (term,)):
+ if docid in results:
+ results[docid] += weight
+ else:
+ results[docid] = weight
+
+ results = sorted(results.iteritems(), key=lambda tup:tup[1], reverse=True)
+ members = []
+ for docid, weight in islice(results, 24):
+ try:
+ member_id, community_id = execute(u"SELECT dispersy_id, square FROM member WHERE id = ?", (docid,)).next()
+ cid, = dispersy_execute(u"SELECT master.mid FROM community JOIN member AS master ON master.id = community.master WHERE community.id = ?", (community_id,)).next()
+ mid, = dispersy_execute(u"SELECT mid FROM member WHERE id = ?", (member_id,)).next()
+ except StopIteration:
+ if __debug__: dprint("unable to determine results for docid ", docid, level="error", exception=1)
+ continue
+ else:
+ members.append((weight, str(cid), str(mid)))
+
+ if members:
+ responses.append(meta.impl(distribution=(self.global_time,), destination=(message.candidate,), payload=(payload.identifier, members)))
+
+ if responses:
+ self._dispersy.store_update_forward(responses, False, False, True)
+
+ def on_search_square_request(self, messages):
+ dispersy_execute = self._dispersy.database.execute
+ execute = self._database.execute
+ meta = self._meta_messages[u"search-square-response"]
+ responses = []
+
+ for message in messages:
+ payload = message.payload
+ if __debug__: dprint("searching ", " + ".join("%d:%s" % (weight, term) for weight, term in payload.terms), " for ", message.candidate)
+
+ results = dict()
+ for weight, term in payload.terms:
+ for docid, in execute(u"SELECT docid FROM square_fts WHERE title MATCH ?", (term,)):
+ if docid in results:
+ results[docid] += weight
+ else:
+ results[docid] = weight
+
+ for docid, in execute(u"SELECT docid FROM square_fts WHERE description MATCH ?", (term,)):
+ if docid in results:
+ results[docid] += weight
+ else:
+ results[docid] = weight
+
+ results = sorted(results.iteritems(), key=lambda tup:tup[1], reverse=True)
+ squares = []
+ for docid, weight in islice(results, 48):
+ try:
+ cid, = dispersy_execute(u"SELECT master.mid FROM community JOIN member AS master ON master.id = community.master WHERE community.id = ?", (docid,)).next()
+ except StopIteration:
+ if __debug__: dprint("unable to determine results for docid ", docid, level="error")
+ continue
+ else:
+ squares.append((weight, str(cid)))
+
+ if squares:
+ responses.append(meta.impl(distribution=(self.global_time,), destination=(message.candidate,), payload=(payload.identifier, squares,)))
- # TODO currently always responding with whats hot
- if self._implicitly_hot_text:
- hots = [Hot(message.community.cid, msg.authentication.member.mid, msg.distribution.global_time) for msg in self._implicitly_hot_text[:10]]
+ if responses:
+ self._dispersy.store_update_forward(responses, False, False, True)
- response = meta.impl(distribution=(self.global_time,), destination=(message.candidate,), payload=(hots,))
- self._dispersy.store_update_forward([response], False, False, True)
- if __debug__: dprint("responding with ", len(hots), " hot messages")
+ def on_search_text_request(self, messages):
+ dispersy_execute = self._dispersy.database.execute
+ execute = self._database.execute
+ meta = self._meta_messages[u"search-text-response"]
+ responses = []
+
+ for message in messages:
+ payload = message.payload
+ if __debug__: dprint("searching ", " + ".join("%d:%s" % (weight, term) for weight, term in payload.terms), " for ", message.candidate)
+
+ results = dict()
+ for weight, term in payload.terms:
+ for docid, in execute(u"SELECT docid FROM text_fts WHERE text MATCH ?", (term,)):
+ if docid in results:
+ results[docid] += weight
+ else:
+ results[docid] = weight
+
+ results = sorted(results.iteritems(), key=lambda tup:tup[1], reverse=True)
+ texts = []
+ for docid, weight in islice(results, 20):
+ try:
+ cid, mid, global_time = dispersy_execute(u"SELECT master.mid, member.mid, sync.global_time FROM sync JOIN community ON community.id = sync.community JOIN member AS master ON master.id = community.master JOIN member ON member.id = sync.member WHERE sync.id = ?", (docid,)).next()
+ except StopIteration:
+ if __debug__: dprint("unable to determine results for docid ", docid, level="error")
+ continue
+ else:
+ texts.append((weight, str(cid), str(mid), global_time))
+
+ if texts:
+ responses.append(meta.impl(distribution=(self.global_time,), destination=(message.candidate,), payload=(payload.identifier, texts,)))
+
+ if responses:
+ self._dispersy.store_update_forward(responses, False, False, True)
+
+ def on_search_member_response(self, messages):
+ pass
+
+ def on_search_square_response(self, messages):
+ pass
- def on_search_response(self, messages):
+ def on_search_text_response(self, messages):
pass
View
132 src/discovery/conversion.py
@@ -9,10 +9,12 @@ class Conversion(BinaryConversion):
def __init__(self, community):
super(Conversion, self).__init__(community, "\x01")
self.define_meta_message(chr(1), community.get_meta_message(u"hots"), self._encode_hots, self._decode_hots)
- self.define_meta_message(chr(2), community.get_meta_message(u"search"), self._encode_search, self._decode_search)
- # TODO the search-response should include a response identifier
- self.define_meta_message(chr(3), community.get_meta_message(u"search-response"), self._encode_hots, self._decode_hots)
-
+ self.define_meta_message(chr(2), community.get_meta_message(u"search-member-request"), self._encode_search_request, self._decode_search_request)
+ self.define_meta_message(chr(3), community.get_meta_message(u"search-member-response"), self._encode_search_member_response, self._decode_search_member_response)
+ self.define_meta_message(chr(4), community.get_meta_message(u"search-square-request"), self._encode_search_request, self._decode_search_request)
+ self.define_meta_message(chr(5), community.get_meta_message(u"search-square-response"), self._encode_search_square_response, self._decode_search_square_response)
+ self.define_meta_message(chr(6), community.get_meta_message(u"search-text-request"), self._encode_search_request, self._decode_search_request)
+ self.define_meta_message(chr(7), community.get_meta_message(u"search-text-response"), self._encode_search_text_request, self._decode_search_text_response)
def _encode_hots(self, message):
return [pack("!20s20sQ", hot.cid, hot.mid, hot.global_time) for hot in message.payload.hots]
@@ -31,24 +33,114 @@ def _decode_hots(self, placeholder, offset, data):
return offset, placeholder.meta.payload.implement(hots)
- def _encode_search(self, message):
- expression = message.payload.expression.encode("UTF-8")
- return pack("!H", min(len(expression), 1024-1)), expression[:1024-1]
+ def _encode_search_request(self, message):
+ payload = message.payload
+ data = [pack("!HBB", payload.identifier, payload.threshold, len(payload.terms))]
+ for weight, term in payload.terms:
+ term = term.encode("UTF-8")
+ data.append(pack("!bB", weight, len(term)))
+ data.append(term)
+ data.append(pack("!B", len(payload.squares)))
+ for weight, cid in payload.squares:
+ data.append(pack("!b20s", weight, cid))
+ return data
+
+ def _decode_search_request(self, placeholder, offset, data):
+ if len(data) < offset + 4:
+ raise DropPacket("Insufficient packet size")
+ identifier, threshold, term_length = unpack_from("!HBB", data, offset)
+ offset += 4
+
+ terms = []
+ for _ in xrange(term_length):
+ if len(data) < offset + 2:
+ raise DropPacket("Insufficient packet size")
+ weight, term_length = unpack_from("!bB", data, offset)
+ offset += 2
+
+ if len(data) < offset + term_length:
+ raise DropPacket("Insufficient packet size")
+ try:
+ term = data[offset:offset+term_length].decode("UTF-8")
+ except UnicodeError:
+ raise DropPacket("Unable to decode UTF-8")
+ offset += term_length
+
+ terms.append((weight, term))
+
+ if len(data) < offset + 1:
+ raise DropPacket("Insufficient packet size")
+ square_length, = unpack_from("!B", data, offset)
+ offset += 1
+
+ squares = []
+ for _ in xrange(square_length):
+ if len(data) < offset + 21:
+ raise DropPacket("Insufficient packet size")
+ squares.append(unpack_from("!b20s", data, offset))
+ offset += 21
+
+ return offset, placeholder.meta.payload.implement(identifier, terms, squares, threshold)
- def _decode_search(self, placeholder, offset, data):
- if len(data) < offset + 2:
+ def _encode_search_member_response(self, message):
+ payload = message.payload
+ data = [pack("!HB", payload.identifier, len(payload.members))]
+ data.extend(pack("!B20s20s", weight, cid, mid) for weight, cid, mid in payload.members)
+ return data
+
+ def _encode_search_square_response(self, message):
+ payload = message.payload
+ data = [pack("!HB", payload.identifier, len(payload.squares))]
+ data.extend(pack("!B20s", weight, cid) for weight, cid in payload.squares)
+ return data
+
+ def _encode_search_text_request(self, message):
+ payload = message.payload
+ data = [pack("!HB", payload.identifier, len(payload.texts))]
+ data.extend(pack("!B20s20sQ", weight, cid, mid, global_time) for weight, cid, mid, global_time in payload.texts)
+ return data
+
+ def _decode_search_member_response(self, placeholder, offset, data):
+ if len(data) < offset + 3:
raise DropPacket("Insufficient packet size")
- expression_length, = unpack_from("!H", data, offset)
- if not expression_length < 1024:
- raise DropPacket("invalid description_length")
- offset += 2
+ identifier, count = unpack_from("!HB", data, offset)
+ offset += 3
- if len(data) < offset + expression_length:
+ members = []
+ for _ in xrange(count):
+ if len(data) < offset + 41:
+ raise DropPacket("Insufficient packet size")
+ members.append(unpack_from("!B20s20s", data, offset))
+ offset += 41
+
+ return offset, placeholder.meta.payload.implement(identifier, members)
+
+ def _decode_search_square_response(self, placeholder, offset, data):
+ if len(data) < offset + 3:
raise DropPacket("Insufficient packet size")
- try:
- expression = data[offset:offset+expression_length].decode("UTF-8")
- except UnicodeError:
- raise DropPacket("Unable to decode UTF-8")
- offset += expression_length
+ identifier, count = unpack_from("!HB", data, offset)
+ offset += 3
+
+ squares = []
+ for _ in xrange(count):
+ if len(data) < offset + 21:
+ raise DropPacket("Insufficient packet size")
+ squares.append(unpack_from("!B20s", data, offset))
+ offset += 21
+
+ return offset, placeholder.meta.payload.implement(identifier, squares)
+
+ def _decode_search_text_response(self, placeholder, offset, data):
+ if len(data) < offset + 3:
+ raise DropPacket("Insufficient packet size")
+ identifier, count = unpack_from("!HB", data, offset)
+ offset += 3
+
+ texts = []
+ for _ in xrange(count):
+ if len(data) < offset + 49:
+ raise DropPacket("Insufficient packet size")
+ texts.append(unpack_from("!B20s20sQ", data, offset))
+ offset += 49
- return offset, placeholder.meta.payload.implement(expression)
+ return offset, placeholder.meta.payload.implement(identifier, texts)
View
84 src/discovery/payload.py
@@ -10,24 +10,76 @@ def __init__(self, meta, hots):
def hots(self):
return self._hots
-class SearchPayload(Payload):
+class SearchRequestPayload(Payload):
class Implementation(Payload.Implementation):
- def __init__(self, meta, expression):
- assert isinstance(expression, unicode)
- assert len(expression) < 1024
- super(SearchPayload.Implementation, self).__init__(meta)
- self._expression = expression
+ def __init__(self, meta, identifier, terms, squares, threshold):
+ """
+ Implement search request.
- @property
- def expression(self):
- return self._expression
+ IDENTIFIER is a 2 byte number.
+ TERMS is a list containing (weight, string) pairs.
+ SQUARES is a list containing (weight, community-id) pairs.
+ THRESHOLD is the minimal value for a hit to qualify.
+ """
+ assert isinstance(terms, list) and 0 < len(terms) < 256
+ assert all(isinstance(x, tuple) and len(x) == 2 for x in terms)
+ assert all(isinstance(x[0], int) and -128 <= x[0] < 128 for x in terms)
+ assert all(isinstance(x[1], unicode) and len(x[1].encode("UTF-8")) < 256 for x in terms)
+ assert isinstance(squares, list) and len(squares) < 256
+ assert all(isinstance(x, tuple) and len(x) == 2 for x in squares)
+ assert all(isinstance(x[0], int) and -128 <= x[0] < 128 for x in squares)
+ assert all(isinstance(x[1], unicode) and len(x[1]) == 20 for x in squares)
+ assert isinstance(threshold, int) and 0 < threshold < 256
+ super(SearchRequestPayload.Implementation, self).__init__(meta)
+ self.identifier = identifier
+ self.terms = terms
+ self.squares = squares
+ self.threshold = threshold
+
+class SearchMemberRequestPayload(SearchRequestPayload):
+ pass
+
+class SearchSquareRequestPayload(SearchRequestPayload):
+ pass
+
+class SearchTextRequestPayload(SearchRequestPayload):
+ pass
-class SearchResponsePayload(Payload):
+class SearchMemberResponsePayload(Payload):
class Implementation(Payload.Implementation):
- def __init__(self, meta, hots):
- super(SearchResponsePayload.Implementation, self).__init__(meta)
- self._hots = hots
+ def __init__(self, meta, identifier, members):
+ """
+ Implement search response.
- @property
- def hots(self):
- return self._hots
+ IDENTIFIER is a 2 byte number.
+ MEMBERS is a list containing (weight, community-id, member-id) triplets.
+ """
+ super(SearchMemberResponsePayload.Implementation, self).__init__(meta)
+ self.identifier = identifier
+ self.members = members
+
+class SearchSquareResponsePayload(Payload):
+ class Implementation(Payload.Implementation):
+ def __init__(self, meta, identifier, squares):
+ """
+ Implement search response.
+
+ IDENTIFIER is a 2 byte number.
+ SQUARES is a list containing (weight, community-id) pairs.
+ """
+ super(SearchSquareResponsePayload.Implementation, self).__init__(meta)
+ self.identifier = identifier
+ self.squares = squares
+
+class SearchTextResponsePayload(Payload):
+ class Implementation(Payload.Implementation):
+ def __init__(self, meta, identifier, texts):
+ """
+ Implement search response.
+
+ IDENTIFIER is a 2 byte number.
+ TEXTS is a list containing (weight, community-id, member-id, global-time) quadruples.
+ """
+ super(SearchTextResponsePayload.Implementation, self).__init__(meta)
+ self.identifier = identifier
+ self.texts = texts
View
40 src/square/community.py
@@ -68,9 +68,10 @@ def __init__(self, master, discovery):
self._dispersy.database.attach_commit_callback(self._database.commit)
self._discovery = discovery
+ self._my_member_info = self._dispersy.get_last_message(self, self._my_member, self._meta_messages[u"member-info"])
try:
- self._update_global_time, self._title, self._description, thumbnail_hash = self._database.execute(u"SELECT global_time, title, description, thumbnail_hash FROM square WHERE id = ?", (self._database_id,)).next()
+ self._update_global_time, self._title, self._description, thumbnail_hash = self._database.execute(u"SELECT global_time, title, description, thumbnail_hash FROM square JOIN square_fts ON docid = dispersy_id WHERE dispersy_id = ?", (self._database_id,)).next()
self._thumbnail_hash = str(thumbnail_hash)
except StopIteration:
self._update_global_time = 0
@@ -79,19 +80,29 @@ def __init__(self, master, discovery):
self._thumbnail_hash = ""
try:
- self._my_alias, my_thumbnail_hash = self._database.execute(u"SELECT alias, thumbnail_hash FROM member WHERE id = ? AND square = ?", (self._my_member.database_id, self._database_id)).next()
+ self._my_alias, my_thumbnail_hash = self._database.execute(u"SELECT alias, thumbnail_hash FROM member JOIN member_fts ON docid = dispersy_id WHERE dispersy_id = ? AND square = ?", (self._my_member.database_id, self._database_id)).next()
self._my_thumbnail_hash = str(my_thumbnail_hash)
except StopIteration:
self._my_alias = u"Anonymous"
self._my_thumbnail_hash = ""
- if __debug__: dprint("new Square '", self._title, "'. using alias '", self._my_alias, "'")
+ if __debug__: dprint("new Square '", self._title, "' using alias '", self._my_alias, "'")
self.events = getEventBroker(self)
self.global_events = getEventBroker(None)
def load_history():
- for id_, global_time, member_id, member_alias, member_thumbnail_hash, text, media_hash, utc_timestamp in self._database.execute(u"SELECT text.id, text.global_time, text.member, member.alias, member.thumbnail_hash, text.text, text.media_hash, text.utc_timestamp FROM text JOIN member ON member.id = text.member WHERE text.square = ? ORDER BY global_time, utc_timestamp DESC LIMIT 100", (self._database_id,)):
+ sql = u"""
+SELECT text.dispersy_id, text.global_time, member.id, member_fts.alias, member.thumbnail_hash, text_fts.text, text.media_hash, text.utc_timestamp
+FROM text
+JOIN text_fts ON text_fts.docid = text.dispersy_id
+JOIN member ON member.id = text.member
+JOIN member_fts ON member_fts.docid = member.dispersy_id
+WHERE text.square = ?
+ORDER BY text.global_time, text.utc_timestamp DESC
+LIMIT 100"""
+
+ for id_, global_time, member_id, member_alias, member_thumbnail_hash, text, media_hash, utc_timestamp in self._database.execute(sql, (self._database_id,)):
member = Member(self, member_id, member_alias, str(member_thumbnail_hash))
text = Text(self, id_, global_time, member, text, str(media_hash), utc_timestamp)
self.events.messageReceived(text)
@@ -188,19 +199,23 @@ def set_my_member_info(self, name, thumbnail_hash):
def on_member_info(self, messages):
data = []
+ data_fts = []
+
for message in messages:
member = Member(self, message.authentication.member.database_id, message.payload.alias, message.payload.thumbnail_hash)
# database data
- data.append((member.id, self._database_id, member.alias, buffer(member.thumbnail_hash)))
+ data.append((member.id, self._database_id, buffer(member.thumbnail_hash)))
+ data_fts.append((member.id, member.alias))
if member.id == self._my_member.database_id:
self._my_alias = member.alias
self._my_thumbnail_hash = member.thumbnail_hash
self.events.memberInfoUpdated(member)
- self._database.executemany(u"INSERT OR REPLACE INTO member (id, square, alias, thumbnail_hash) VALUES (?, ?, ?, ?)", data)
+ self._database.executemany(u"INSERT OR REPLACE INTO member (dispersy_id, square, thumbnail_hash) VALUES (?, ?, ?)", data)
+ self._database.executemany(u"INSERT OR REPLACE INTO member_fts (docid, alias) VALUES (?, ?)", data_fts)
def undo_member_info(self, *args):
pass
@@ -237,7 +252,8 @@ def on_square_info(self, messages):
self._title = message.payload.title
self._description = message.payload.description
self._thumbnail_hash = message.payload.thumbnail_hash
- self._database.execute(u"INSERT OR REPLACE INTO square (id, global_time, title, description, thumbnail_hash) VALUES (?, ?, ?, ?, ?)", (self._database_id, self._update_global_time, self._title, self._description, buffer(self._thumbnail_hash)))
+ self._database.execute(u"INSERT OR REPLACE INTO square (dispersy_id, global_time, thumbnail_hash) VALUES (?, ?, ?)", (self._database_id, self._update_global_time, buffer(self._thumbnail_hash)))
+ self._database.execute(u"INSERT OR REPLACE INTO square_fts (docid, title, description) VALUES (?, ?, ?)", (self._database.last_insert_rowid, self._title, self._description))
update = True
if update:
@@ -249,7 +265,7 @@ def undo_square_info(self, *args):
def post_text(self, text, media_hash):
if self._my_member_info is None:
- raise ValueError("invalid my member info, set_my_member_info must be called at least once before posting")
+ self.set_my_member_info(self._my_alias, self._my_thumbnail_hash)
if not (isinstance(text, unicode) and len(text.encode("UTF-8")) < 1024):
raise ValueError("invalid text")
if not (isinstance(media_hash, str) and len(media_hash) in (0, 20)):
@@ -268,17 +284,21 @@ def post_text(self, text, media_hash):
def on_text(self, messages, mark_as_hot=True):
data = []
+ data_fts = []
+
for message in messages:
member = Member(self, message.payload.member_info.authentication.member.database_id, message.payload.member_info.payload.alias, message.payload.member_info.payload.thumbnail_hash)
text = Text(self, message.packet_id, message.distribution.global_time, member, message.payload.text, message.payload.media_hash, message.payload.utc_timestamp)
# database data
- data.append((text.id, text.global_time, self._database_id, member.id, text.text, buffer(text.media_hash), text.utc_timestamp))
+ data.append((text.id, text.global_time, self._database_id, member.id, buffer(text.media_hash), text.utc_timestamp))
+ data_fts.append((text.id, text.text))
# update GUI: message has been received
self.events.messageReceived(text)
- self._database.executemany(u"INSERT INTO text (id, global_time, square, member, text, media_hash, utc_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)", data)
+ self._database.executemany(u"INSERT INTO text (dispersy_id, global_time, square, member, media_hash, utc_timestamp) VALUES (?, ?, ?, ?, ?, ?)", data)
+ self._database.executemany(u"INSERT INTO text_fts (docid, text) VALUES (?, ?)", data_fts)
if mark_as_hot:
self._discovery.add_implicitly_hot_text(messages)
View
21 src/square/database.py
@@ -6,29 +6,30 @@
schema = u"""
CREATE TABLE square(
- id INTEGER, -- the dispersy Community.database_id
+ dispersy_id INTEGER, -- the dispersy Community.database_id
global_time INTEGER, -- the dispersy Message.distribution.global_time
- title TEXT,
- description TEXT,
thumbnail_hash BLOB,
- PRIMARY KEY (id));
+ PRIMARY KEY (dispersy_id));
CREATE TABLE member(
- id INTEGER, -- the dispersy Member.database_id
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ dispersy_id INTEGER, -- the dispersy Member.database_id
square INTEGER REFERENCES square(id),
- alias TEXT,
thumbnail_hash BLOB,
- PRIMARY KEY (id, square));
+ UNIQUE (id, square));
CREATE TABLE text(
- id INTEGER, -- the dispersy Message.database_id
+ dispersy_id INTEGER, -- the dispersy Message.database_id
global_time INTEGER, -- the dispersy Message.distribution.global_time
square INTEGER REFERENCES square(id),
member INTEGER REFERENCES member(id),
- text TEXT,
media_hash BLOB,
utc_timestamp INTEGER,
- PRIMARY KEY (id));
+ PRIMARY KEY (dispersy_id));
+
+CREATE VIRTUAL TABLE square_fts USING fts4(title, description, tokenize=porter);
+CREATE VIRTUAL TABLE member_fts USING fts4(alias, tokenize=porter);
+CREATE VIRTUAL TABLE text_fts USING fts4(text, tokenize=porter);
CREATE TABLE option(key TEXT PRIMARY KEY, value BLOB);
INSERT INTO option(key, value) VALUES('database_version', '""" + str(LATEST_VERSION) + """');

0 comments on commit 107e241

Please sign in to comment.
Something went wrong with that request. Please try again.