-
Notifications
You must be signed in to change notification settings - Fork 35
/
live.py
1712 lines (1449 loc) · 87.4 KB
/
live.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
"""
Functionality for communicating with Skype online service.
------------------------------------------------------------------------------
This file is part of Skyperious - Skype chat history tool.
Released under the MIT License.
@author Erki Suurjaak
@created 08.07.2020
@modified 02.06.2024
------------------------------------------------------------------------------
"""
import collections
import datetime
import glob
import inspect
import json
import logging
import os
import re
import struct
import sys
import tarfile
import tempfile
import time
import warnings
BeautifulSoup = skpy = ijson = None
try: from bs4 import BeautifulSoup
except ImportError: pass
try: import skpy
except ImportError: pass
try: import ijson
except ImportError: pass
import six
from six.moves import urllib
from . lib import util
from . import conf
from . import skypedata
logger = logging.getLogger(__name__)
# Avoid BeautifulSoup popup warnings like MarkupResemblesLocatorWarning
warnings.filterwarnings("ignore", category=UserWarning, module="bs4")
warnings.filterwarnings("ignore", category=UserWarning, module="skpy")
class SkypeLogin(object):
"""
Class for logging into Skype web account and retrieving chat history.
"""
# Enum for save() results
SAVE = type('', (), dict(SKIP=0, INSERT=1, UPDATE=2, NOCHANGE=3))
# Relevant columns in Messages-table
CACHE_COLS = {"messages": [
"author", "body_xml", "chatmsg_type", "convo_id", "dialog_partner",
"edited_by", "edited_timestamp", "from_dispname", "guid", "id",
"identities", "is_permanent", "pk_id", "remote_id", "timestamp",
"timestamp__ms", "type"
]}
def __init__(self, db=None, progress=None):
"""
@param db existing SkypeDatabase instance if not creating new
@param progress callback function invoked with (action, ?error, ?table, ?count, ?total, ?new, ?updated),
returning false if work should stop
"""
self.db = db # SQLite database populated with live data
self.progress = progress or (lambda *_, **__: True)
self.username = db.username if db else None
self.skype = None # skpy.Skype instance
self.tokenpath = None # Path to login tokenfile
self.cache = collections.defaultdict(dict) # {table: {identity: {item}}}
self.dl_cache_dir = os.path.join(conf.CacheDirectory, "shared")
self.populated = False
self.sync_counts = collections.defaultdict(int) # {"contacts_new": x, ..}
self.query_stamps = [] # [datetime.datetime, ] for rate limiting
self.msg_stamps = {} # {remote_id: last timestamp__ms}
self.msg_lookups = collections.defaultdict(list) # {timestamp__ms: [{msg}, ]}
self.msg_parser = None # skypedata.MessageParser instance
def is_logged_in(self):
"""Returns whether there is active login."""
return bool(self.skype and self.skype.conn.connected)
def login(self, username=None, password=None, token=True, init_db=True):
"""
Logs in to Skype with given username or raises error on failure.
Can try to use existing tokenfile if available (tokenfiles expire in 24h).
Creates database if not already created.
@param token whether to use existing tokenfile instead of password
@param init_db whether to create and open Skype database after login
"""
if self.db and self.db.username: self.username = self.db.username
if username and not self.username: self.username = username
path = util.safe_filename(self.username)
if path != self.username: path += "_%x" % util.hash_string(self.username)
path = self.tokenpath = os.path.join(conf.VarDirectory, "%s.token" % path)
logger.info("Logging in to Skype online service as '%s'.", self.username)
kwargslist = []
if password: kwargslist += [{"user": self.username, "pwd": password, "tokenFile": path}]
if token and os.path.isfile(path) and os.path.getsize(path):
kwargslist.insert(0, {"tokenFile": path}) # Try with existing token
else:
util.create_file(path)
for kwargs in kwargslist:
try: self.skype = skpy.Skype(**kwargs)
except Exception:
if kwargs is not kwargslist[-1]: continue # for kwargs
_, e, tb = sys.exc_info()
if self.username and password and "@" not in self.username:
# Special case: if a legacy account is linked to a MS account,
# it needs to use soapLogin() explicitly
# (skpy auto-selects liveLogin() for legacy accounts).
sk = skpy.Skype(tokenFile=path, connect=False)
try: sk.conn.soapLogin(self.username, password)
except Exception: _, e, tb = sys.exc_info()
else:
self.skype = sk
break # for kwargs
logger.exception("Error logging in to Skype as '%s'.", self.username)
try: os.unlink(path)
except Exception: pass
six.reraise(type(e), e, tb)
else: break # for kwargs
if init_db: self.init_db()
def init_db(self, filename=None, truncate=False):
"""Creates SQLite database if not already created."""
if not self.db:
path = filename or make_db_path(self.username)
truncate = truncate or not os.path.exists(path)
self.db = skypedata.SkypeDatabase(path, truncate=truncate)
self.db.live = self
self.db.ensure_schema(create_only=True)
self.msg_parser = skypedata.MessageParser(self.db)
def build_cache(self, *tables):
"""
Fills in local cache.
@param tables specific tables to build if not all
"""
BINARIES = "avatar_image", "guid", "meta_picture"
tables = [x.lower() for x in tables]
if tables:
for table in tables:
self.cache.pop(table, None)
if "messages" == table:
self.msg_lookups.clear(), self.msg_stamps.clear()
else:
for dct in (self.cache, self.msg_lookups, self.msg_stamps): dct.clear()
for table in "accounts", "chats", "contacts":
if tables and table not in tables: continue # for table
key = "identity" if "chats" == table else "skypename"
cols = ", ".join(self.CACHE_COLS[table]) if table in self.CACHE_COLS else "*"
dbtable = "conversations" if "chats" == table else table
where = " WHERE %s IS NOT NULL" % key
for row in self.db.execute("SELECT %s FROM %s%s" % (cols, dbtable, where), log=False):
for k in BINARIES: # Convert binary fields back from Unicode
if isinstance(row.get(k), six.text_type):
try:
row[k] = row[k].encode("latin1")
except Exception:
row[k] = row[k].encode("utf-8")
self.cache[table][row[key]] = row
self.cache["contacts"].update(self.cache["accounts"]) # For name lookup
def build_msg_cache(self, identity):
"""Fills in message cache for chat."""
BINARIES = ("guid", )
for dct in (self.msg_lookups, self.msg_stamps, self.cache["messages"]):
dct.clear()
chats = self.db.get_conversations(chatidentities=[identity], reload=True, log=False)
if not chats: return
cc = [x for x in (chats[0], chats[0].get("__link")) if x]
ff = [":convo_id%s" % i for i in range(len(cc))]
where = " WHERE convo_id IN (%s)" % ", ".join(ff)
params = {"convo_id%s" % i: c["id"] for i, c in enumerate(cc)}
table, key = "messages", "pk_id"
cols = ", ".join(self.CACHE_COLS[table]) if table in self.CACHE_COLS else "*"
for row in self.db.execute("SELECT %s FROM %s%s" % (cols, table, where), params, log=False):
for k in BINARIES: # Convert binary fields back from Unicode
if isinstance(row.get(k), six.text_type):
try:
row[k] = row[k].encode("latin1")
except Exception:
row[k] = row[k].encode("utf-8")
if row[key] is not None: self.cache[table][row[key]] = row
if row["timestamp__ms"] is not None:
self.msg_lookups[row["timestamp__ms"]].append(row)
for m in self.cache["messages"].values():
if m.get("remote_id") is not None and m.get("timestamp__ms") is not None:
val = max(m["timestamp__ms"], self.msg_stamps.get(m["remote_id"], -sys.maxsize))
self.msg_stamps[m["remote_id"]] = val
def request(self, func, *args, **kwargs):
"""
Invokes Skype request function with positional and keyword arguments,
handles request rate limiting and transient communication errors.
@param __retry special keyword argument, does not retry if falsy
@param __raise special keyword argument, does not raise if falsy
@param __log special keyword argument, does not log error if falsy
"""
self.query_stamps.append(datetime.datetime.now())
while len(self.query_stamps) > conf.LiveSyncRateLimit:
self.query_stamps.pop(0)
dts = self.query_stamps
delta = (dts[-1] - dts[0]) if len(dts) >= conf.LiveSyncRateLimit else 0
if isinstance(delta, datetime.timedelta) and delta.total_seconds() < conf.LiveSyncRateWindow:
time.sleep(conf.LiveSyncRateWindow - delta.total_seconds())
doretry, doraise, dolog = (kwargs.pop(k, True) for k in ("__retry", "__raise", "__log"))
tries = 0
while True:
try: return func(*args, **kwargs)
except Exception as e:
tries += 1
if tries > conf.LiveSyncRetryLimit or not doretry:
if dolog: logger.exception("Error calling %r.", func)
if doraise: raise
return None
delay = conf.LiveSyncRetryDelay
try: code = e.args[1].status_code
except Exception: code = None
if 429 == code: # TOO_MANY_REQUESTS
delay = conf.LiveSyncAuthRateLimitDelay
logger.debug("Hit Skype online rate limit when calling %r, "
"sleeping %s seconds.\n%r", func, delay, e)
time.sleep(delay)
finally: # Replace with final
self.query_stamps[-1] = datetime.datetime.now()
def reqattr(self, obj, name):
"""Returns Skype object attribute value, channeling via request() if class property."""
prop = getattr(type(obj), name, None)
return self.request(prop.fget, obj) if inspect.isdatadescriptor(prop) else getattr(obj, name)
def save(self, table, item, parent=None):
"""
Saves the item to SQLite table.
@param table database table to save into
@param item Skype API data object
@param parent chat for messages
@return (database row, one of SkypeLogin.SAVE)
"""
if item is None: return (None, self.SAVE.SKIP)
dbitem = self.convert(table, item, parent=parent)
if dbitem is None: return (None, self.SAVE.SKIP)
result, dbitem1, table = None, dict(dbitem), table.lower()
identity = dbitem["skypename"] if table in ("accounts", "contacts") else \
dbitem["identity"] if "chats" == table else dbitem.get("pk_id")
dbitem0 = self.cache[table].get(identity)
if "contacts" == table and not dbitem0 \
and not (identity or "").startswith(skypedata.ID_PREFIX_BOT):
# Bot accounts from live can be without bot prefix
dbitem0 = self.cache[table].get(skypedata.ID_PREFIX_BOT + identity)
if dbitem0:
identity = skypedata.ID_PREFIX_BOT + identity
dbitem["skypename"] = dbitem1["skypename"] = identity
dbitem["type"] = dbitem1["type"] = skypedata.CONTACT_TYPE_BOT
if "contacts" == table and not dbitem0 \
and (identity or "").startswith(skypedata.ID_PREFIX_BOT):
# Fix bot entries in database not having bot prefix
dbitem0 = self.cache[table].get(identity[len(skypedata.ID_PREFIX_BOT):])
if "messages" == table and dbitem.get("remote_id") \
and not isinstance(item, (skpy.SkypeCallMsg, skpy.SkypeMemberMsg, skpy.msg.SkypePropertyMsg)):
# Look up message by remote_id instead, to detect edited messages
dbitem0 = next((v for v in self.cache[table].values()
if v.get("remote_id") == dbitem["remote_id"]
and v["convo_id"] == dbitem["convo_id"]), dbitem0)
if "messages" == table and not dbitem0:
# See if there is a message with same data in key fields
MATCH = ["type", "author", "identities"]
candidates = [v for v in self.msg_lookups.get(dbitem["timestamp__ms"], [])
if all(v.get(k) == dbitem.get(k) for k in MATCH)]
dbitem0 = next((v for v in candidates
if v["body_xml"] == dbitem["body_xml"]), None)
if not dbitem0 and candidates:
# body_xml can differ by whitespace formatting: match formatted
parse_options = {"format": "text", "merge": True}
as_text = lambda m: self.msg_parser.parse(m, output=parse_options)
b2 = as_text(dbitem)
dbitem0 = next((v for v in candidates if as_text(v) == b2), None)
if "chats" == table and isinstance(item, skpy.SkypeGroupChat) \
and not dbitem.get("displayname"):
if dbitem0 and dbitem0["displayname"]:
dbitem["displayname"] = dbitem0["displayname"]
else: # Assemble name from participants
names = []
for uid in self.reqattr(item, "userIds")[:4]: # Use up to 4 names
name = self.get_contact_name(uid)
if not self.get_contact(uid) \
or conf.Login.get(self.db.filename, {}).get("sync_contacts", True):
user = self.request(self.skype.contacts.__getitem__, uid, __raise=False)
if user and user.name: name = util.to_unicode(user.name)
names.append(name)
dbitem["displayname"] = ", ".join(names)
if len(self.reqattr(item, "userIds")) > 4: dbitem["displayname"] += ", ..."
dbitem1 = dict(dbitem)
dbtable = "conversations" if "chats" == table else table
if dbitem0:
dbitem["id"] = dbitem1["id"] = dbitem0["id"]
if "messages" == table and dbitem0.get("remote_id") == dbitem.get("remote_id") \
and (dbitem0["pk_id"] != dbitem["pk_id"] or dbitem0["body_xml"] != dbitem["body_xml"]) \
and not (isinstance(item, skpy.SkypeFileMsg) # body_xml contains url, starting from v4.9
and dbitem0["pk_id"] == dbitem["pk_id"]
and dbitem0["timestamp__ms"] == dbitem["timestamp__ms"]):
# Different messages with same remote_id -> edited or deleted message
dbitem["edited_by"] = dbitem["author"]
dbitem["edited_timestamp"] = max(dbitem["timestamp"], dbitem0["timestamp"])
dbitem["edited_timestamp"] = max(dbitem["edited_timestamp"], (dbitem0.get("edited_timestamp") or 0))
self.msg_stamps[dbitem["remote_id"]] = max(dbitem["timestamp__ms"],
self.msg_stamps.get(dbitem["remote_id"],
-sys.maxsize))
if dbitem["timestamp__ms"] > dbitem0["timestamp__ms"] \
or (dbitem["timestamp__ms"] == dbitem0["timestamp__ms"] and dbitem["pk_id"] > dbitem0["pk_id"]):
dbitem.update({k: dbitem0[k] if dbitem0[k] is not None else dbitem[k]
for k in ("pk_id", "guid", "timestamp", "timestamp__ms")})
else:
dbitem["body_xml"] = dbitem0["body_xml"]
dbitem1 = dict(dbitem)
self.db.update_row(dbtable, dbitem, dbitem0, log=False)
for k, v in dbitem0.items():
if k not in dbitem: dbitem[k] = v
dbitem["__updated__"] = True
if "contacts" == table and identity.startswith(skypedata.ID_PREFIX_BOT) \
and not dbitem0["skypename"].startswith(skypedata.ID_PREFIX_BOT):
# Fix bot entries in database not having bot prefix
self.db.execute("UPDATE participants SET identity = :id WHERE identity = :id0",
{"id": identity, "id0": dbitem0["skypename"]})
self.db.execute("UPDATE transfers SET partner_handle = :id WHERE partner_handle = :id0",
{"id": identity, "id0": dbitem0["skypename"]})
self.db.execute("UPDATE messages SET author = :id WHERE author = :id0",
{"id": identity, "id0": dbitem0["skypename"]})
dbitem0["skypename"] = identity
else:
dbitem["id"] = self.db.insert_row(dbtable, dbitem, log=False)
dbitem["__inserted__"] = True
self.sync_counts["%s_new" % table] += 1
result = self.SAVE.INSERT
if "messages" == table and dbitem1.get("remote_id") is not None \
and dbitem1["remote_id"] not in self.msg_stamps:
self.msg_stamps[dbitem1["remote_id"]] = dbitem1["timestamp__ms"]
if identity is not None:
cacheitem = dict(dbitem)
self.cache[table][identity] = cacheitem
if "accounts" == table:
self.cache["contacts"][identity] = cacheitem # For name lookup
if "messages" == table:
self.msg_lookups[dbitem["timestamp__ms"]].append(cacheitem)
if "chats" == table:
self.insert_participants(item, dbitem, dbitem0)
self.insert_chats(item, dbitem, dbitem0)
if "chats" == table and isinstance(item, skpy.SkypeGroupChat):
# See if older-style chat entry is present
identity0 = id_to_identity(item.id)
chat0 = self.cache["chats"].get(identity0) if identity0 != item.id else None
if chat0:
result = self.SAVE.NOCHANGE
# Use type() instead of isinstance() as image/audio/video are handled differently
if "messages" == table and type(item) is skpy.SkypeFileMsg and item.file:
self.insert_transfers(item, dbitem, dbitem0)
if "messages" == table and isinstance(item, skpy.SkypeCallMsg):
self.insert_calls(item, dbitem, dbitem0)
if result is None and dbitem0 and dbitem0.get("__inserted__"):
result = self.SAVE.SKIP # Primary was inserted during this run, no need to count
if result is None and dbitem0:
with warnings.catch_warnings():
warnings.simplefilter("ignore") # Swallow Unicode equality warnings
same = all(v == dbitem0[k] for k, v in dbitem1.items() if k in dbitem0)
result = self.SAVE.NOCHANGE if same else self.SAVE.UPDATE
if not same: self.sync_counts["%s_updated" % table] += 1
return dbitem, result
def insert_transfers(self, msg, row, row0):
"""Inserts Transfers-row for SkypeFileMsg, if not already present."""
try:
t = dict(is_permanent=1, filename=msg.file.name, convo_id=row["convo_id"],
filesize=msg.file.size, starttime=row["timestamp"],
chatmsg_guid=row["guid"], chatmsg_index=0,
type=skypedata.TRANSFER_TYPE_OUTBOUND,
partner_handle=self.prefixify(msg.userId),
partner_dispname=self.get_contact_name(msg.userId))
args = {"convo_id": (row0 or row)["convo_id"], "chatmsg_guid": (row0 or row)["guid"]}
args = self.db.blobs_to_binary(args, list(args),
self.db.get_table_columns("transfers"))
cursor = self.db.execute("SELECT 1 FROM transfers WHERE "
"convo_id = :convo_id "
"AND chatmsg_guid = :chatmsg_guid", args, log=False)
existing = cursor.fetchone()
if not existing: self.db.insert_row("transfers", t, log=False)
except Exception:
logger.exception("Error inserting Transfers-row for message %r.", msg)
def insert_participants(self, chat, row, row0):
"""Inserts Participants-rows for SkypeChat, if not already present."""
try:
participants, savecontacts = [], []
uids = self.reqattr(chat, "userIds") if isinstance(chat, skpy.SkypeGroupChat) \
else set([self.skype.userId, chat.userId])
for uid in uids:
user = self.request(self.skype.contacts.__getitem__, uid)
uidentity = user.id if user else uid
# Keep bot prefixes on bot accounts
if uidentity != self.skype.userId and not uidentity.startswith(skypedata.ID_PREFIX_BOT) \
and (is_bot(user) or chat.id.startswith(skypedata.ID_PREFIX_BOT)):
uidentity = skypedata.ID_PREFIX_BOT + uidentity
p = dict(is_permanent=1, convo_id=row["id"], identity=uidentity)
if isinstance(chat, skpy.SkypeGroupChat) and (user.id if user else uid) == chat.creatorId:
p.update(rank=1)
participants.append(p)
existing = set()
if row0:
cursor = self.db.execute("SELECT identity FROM participants "
"WHERE convo_id = :id", row0, log=False)
existing.update(x["identity"] for x in cursor)
for p in participants:
if p["identity"] not in existing:
self.db.insert_row("participants", p, log=False)
if p["identity"] != self.skype.userId \
and (p["identity"] not in self.cache["contacts"]
or (conf.Login.get(self.db.filename, {}).get("sync_contacts", True)
and not self.cache["contacts"][p["identity"]].get("__updated__")
)):
idx = len(skypedata.ID_PREFIX_BOT) \
if p["identity"].startswith(skypedata.ID_PREFIX_BOT) else 0
uidentity = p["identity"][idx:]
contact = self.request(self.skype.contacts.__getitem__,
uidentity, __raise=False)
if contact: savecontacts.append(contact)
for c in savecontacts: self.save("contacts", c)
except Exception:
logger.exception("Error inserting Participants-rows for chat %r.", chat)
def insert_chats(self, chat, row, row0):
"""Inserts Chats-row for SkypeChat, if not already present."""
try:
if self.db.execute("SELECT 1 FROM chats "
"WHERE conv_dbid = :id", row, log=False).fetchone():
return
cursor = self.db.execute("SELECT identity FROM participants "
"WHERE convo_id = :id", row, log=False)
memberstr = " ".join(sorted(x["identity"] for x in cursor))
c = dict(is_permanent=1, name=row["identity"], conv_dbid=row["id"],
posters=memberstr, participants=memberstr,
activemembers=memberstr, friendlyname=row["displayname"])
self.db.insert_row("chats", c, log=False)
except Exception:
logger.exception("Error inserting Chats-row for chat %r.", chat)
def insert_calls(self, msg, row, row0):
"""Inserts Calls-row for SkypeCallMsg."""
try:
if row0 or skpy.SkypeCallMsg.State.Started == msg.state: return
duration = 0
bs = BeautifulSoup(msg.content, "html.parser") if BeautifulSoup else None
for tag in bs.find_all("duration") if bs else ():
try: duration = max(duration, int(float(tag.text)))
except Exception: pass
ts = row["timestamp"] - duration
c = dict(conv_dbid=row["convo_id"], begin_timestamp=ts,
name="1-%s" % ts, duration=duration)
self.db.insert_row("calls", c, log=False)
except Exception:
logger.exception("Error inserting Calls-row for message %r.", msg)
def convert(self, table, item, parent=None):
"""
Converts item from SkPy object to Skype database dict.
"""
result, table = {}, table.lower()
if table in ("accounts", "contacts"):
# SkypeContact(id='username', name=Name(first='first', last='last'), location=Location(country='EE'), language='ET', avatar='https://avatar.skype.com/v1/avatars/username/public', birthday=datetime.date(1984, 5, 9))
result.update(is_permanent=1, skypename=item.id, languages=item.language)
if item.name:
name = util.to_unicode(item.name)
result.update(displayname=name, fullname=name)
if getattr(item, "birthday", None): # Not all SkypeUsers have birthday
result.update(birthday=date_to_integer(item.birthday))
if item.location and item.location.country:
result.update(country=item.location.country)
if item.location and item.location.region:
result.update(province=item.location.region)
if item.location and item.location.city:
result.update(city=item.location.city)
for phone in getattr(item, "phones", ()) or ():
if skpy.SkypeContact.Phone.Type.Mobile == phone.type:
result.update(phone_mobile=phone.number)
elif skpy.SkypeContact.Phone.Type.Work == phone.type:
result.update(phone_office=phone.number)
elif skpy.SkypeContact.Phone.Type.Home == phone.type:
result.update(phone_home=phone.number)
if item.raw.get("homepage"):
result.update(homepage=item.raw["homepage"])
if item.raw.get("gender"):
try: result.update(gender=int(item.raw["gender"]))
except Exception: pass
if item.raw.get("emails"):
result.update(emails=" ".join(item.raw["emails"]))
if item.avatar: # https://avatar.skype.com/v1/avatars/username/public
raw = self.download_content(item.avatar)
# main.db has NULL-byte in front of image binary
if raw: result.update(avatar_image=b"\0" + raw)
if "accounts" == table:
if item.mood:
result.update(mood_text=util.to_unicode(item.mood))
if re.match(".+@.+", self.username or ""):
result.update(liveid_membername=self.username)
elif "chats" == table:
result.update(identity=item.id)
if isinstance(item, skpy.SkypeSingleChat):
# SkypeSingleChat(id='8:username', alerts=True, userId='username')
uidentity = item.userId
if item.id.startswith(skypedata.ID_PREFIX_BOT): uidentity = item.id # Bot chats have bot prefix
result.update(identity=uidentity, type=skypedata.CHATS_TYPE_SINGLE)
if uidentity not in self.cache["contacts"]:
citem = self.request(self.skype.contacts.__getitem__, item.userId, __raise=False)
if citem: self.save("contacts", citem)
result.update(displayname=self.get_contact_name(item.userId))
elif isinstance(item, skpy.SkypeGroupChat):
# SkypeGroupChat(id='19:xyz==@p2p.thread.skype', alerts=True, topic='chat topic', creatorId='username;epid={87e22f7c-d816-ea21-6cb3-05b477922f95}', userIds=['username1', 'username2'], adminIds=['username'], open=False, history=True, picture='https://experimental-api.asm.skype.com/..')
result.update(type=skypedata.CHATS_TYPE_GROUP, meta_topic=item.topic)
if item.topic:
result.update(displayname=item.topic)
if item.creatorId:
creator = re.sub(";.+$", "", item.creatorId)
if creator: result.update(creator=creator)
if item.picture:
# https://api.asm.skype.com/v1/objects/0-weu-d14-abcdef..
raw = self.get_api_content(item.picture, category="avatar", cache=False)
# main.db has NULL-byte in front of image binary
if raw: result.update(meta_picture="\0" + raw.decode("latin1"))
else: result = None
elif "contacts" == table:
result.update(type=skypedata.CONTACT_TYPE_BOT if is_bot(item)
else skypedata.CONTACT_TYPE_NORMAL,
isblocked=getattr(item, "blocked", False),
isauthorized=getattr(item, "authorised", False))
if item.name and item.name.first:
result.update(firstname=item.name.first)
if item.name and item.name.last:
result.update(lastname=item.name.last)
if item.mood:
result.update(mood_text=item.mood.plain, rich_mood_text=item.mood.rich)
if item.avatar:
result.update(avatar_url=item.avatar)
if is_bot(item) and not item.id.startswith(skypedata.ID_PREFIX_BOT):
result.update(skypename=skypedata.ID_PREFIX_BOT + item.id)
elif "messages" == table:
ts__ms = util.datetime_to_millis(item.time)
pk_id, guid = make_message_ids(item.id)
result.update(is_permanent=1, timestamp=ts__ms // 1000, pk_id=pk_id, guid=guid,
author=self.prefixify(item.userId),
from_dispname=self.get_contact_name(item.userId),
body_xml=item.content, timestamp__ms=ts__ms)
if parent:
identity = parent.id if isinstance(parent, skpy.SkypeGroupChat) \
or is_bot(self.reqattr(parent, "user")) \
or result["author"].startswith(skypedata.ID_PREFIX_BOT) else parent.userId
chat = self.cache["chats"].get(identity)
if not chat:
self.save("chats", parent)
chat = self.cache["chats"].get(identity)
result.update(convo_id=chat["id"])
remote_id = item.raw.get("skypeeditedid") or item.clientId
if remote_id: result.update(remote_id=util.hash_string(remote_id))
if isinstance(parent, skpy.SkypeSingleChat):
partner = result["author"] if result["author"] != self.skype.userId else parent.userId
result.update(dialog_partner=partner)
if isinstance(item, skpy.SkypeTextMsg):
# SkypeTextMsg(id='1594466183791', type='RichText', time=datetime.datetime(2020, 7, 11, 11, 16, 16, 392000), clientId='16811922854185209745', userId='username', chatId='8:username', content='<ss type="hi">(wave)</ss>')
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_MESSAGE,
type=skypedata.MESSAGE_TYPE_MESSAGE)
process_message_edit(result)
elif isinstance(item, skpy.SkypeCallMsg):
# SkypeCallMsg(id='1593784617947', type='Event/Call', time=datetime.datetime(2020, 7, 3, 13, 56, 57, 949000), clientId='3666747505059875266', userId='username', chatId='8:username', content='<partlist type="ended" alt="" callId="1593784540478"><part identity="8:username"><name>Display Name</name><duration>84.82</duration></part><part identity="8:username2"><name>Display Name 2</name><duration>84.82</duration></part></partlist>', state=SkypeCallMsg.State.Ended, userIds=['username', '8:username2'], userNames=['Display Name', 'Display Name 2'])
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_SPECIAL2)
if skpy.SkypeCallMsg.State.Ended == item.state:
result.update(type=skypedata.MESSAGE_TYPE_CALL_END)
else:
result.update(type=skypedata.MESSAGE_TYPE_CALL)
elif isinstance(item, skpy.SkypeContactMsg):
# SkypeContactMsg(id='1594466296388', type='RichText/Contacts', time=datetime.datetime(2020, 7, 11, 11, 18, 9, 509000), clientId='3846577445969002747', userId='username', chatId='8:username', content='<contacts><c t="s" s="username2" f="Display Name"></c></contacts>', contactIds=['username2'], contactNames=['Display Name'])
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_SPECIAL,
type=skypedata.MESSAGE_TYPE_CONTACTS)
elif isinstance(item, skpy.msg.SkypeTopicPropertyMsg):
# SkypeTopicPropertyMsg(id='1594466832242', type='ThreadActivity/TopicUpdate', time=datetime.datetime(2020, 7, 11, 11, 27, 12, 242000), userId='username', chatId='19:abcdef..@thread.skype', content='<topicupdate><eventtime>1594466832367</eventtime><initiator>8:username</initiator><value>The Topic</value></topicupdate>', topic='The Topic')
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_TOPIC,
type=skypedata.MESSAGE_TYPE_TOPIC,
body_xml=item.topic)
elif isinstance(item, skpy.SkypeAddMemberMsg):
# SkypeAddMemberMsg(id='1594467205492', type='ThreadActivity/AddMember', time=datetime.datetime(2020, 7, 11, 11, 33, 25, 492000), userId='username', chatId='19:abcdef..@thread.skype', content='<addmember><eventtime>1594467205492</eventtime><initiator>8:username</initiator><target>8:username2</target></addmember>', memberId='username2')
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_CONTACTS,
type=skypedata.MESSAGE_TYPE_PARTICIPANTS,
identities=self.prefixify(item.memberId))
elif isinstance(item, skpy.SkypeRemoveMemberMsg):
# SkypeRemoveMemberMsg(id='1594467133414', type='ThreadActivity/DeleteMember', time=datetime.datetime(2020, 7, 11, 11, 32, 13, 414000), userId='username', chatId='19:abcdef..@thread.skype', content='<deletemember><eventtime>1594467133727</eventtime><initiator>8:username</initiator><target>8:username2</target></deletemember>', memberId='username2')
if item.userId == item.memberId:
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_LEAVE,
type=skypedata.MESSAGE_TYPE_LEAVE)
else:
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_REMOVE,
type=skypedata.MESSAGE_TYPE_REMOVE,
identities=self.prefixify(item.memberId))
elif isinstance(item, skpy.SkypeLocationMsg):
# SkypeLocationMsg(id='1594466687344', type='RichText/Location', time=datetime.datetime(2020, 7, 11, 11, 24, 0, 305000), clientId='12438271979076057148', userId='username', chatId='8:username', content='<location isUserLocation="0" latitude="59436810" longitude="24740695" timeStamp="1594466640235" timezone="Europe/Tallinn" locale="en-US" language="en" address="Kesklinn, Tallinn, 10146 Harju Maakond, Estonia" addressFriendlyName="Kesklinn, Tallinn, 10146 Harju Maakond, Estonia" shortAddress="Kesklinn, Tallinn, 10146 Harju Maakond, Estonia" userMri="8:username"><a href="https://www.bing.com/maps/default.aspx?cp=9.43681~24.740695&dir=0&lvl=15&where1=Kesklinn,%20Tallinn,%2010137%20Harju%20Maakond,%20Estonia">Kesklinn, Tallinn, 10146 Harju Maakond, Estonia</a></location>', latitude=59.43681, longitude=24.740695, address='Kesklinn, Tallinn, 10146 Harju Maakond, Estonia', mapUrl='https://www.bing.com/maps/default.aspx?cp=59.43681~24.740695&dir=0&lvl=15&where1=Kesklinn,%20Tallinn,%2010146 %20Harju%20Maakond,%20Estonia')
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_SPECIAL,
type=skypedata.MESSAGE_TYPE_INFO)
elif "ThreadActivity/PictureUpdate" == item.type:
# SkypeMsg(id='1594466869805', type='ThreadActivity/PictureUpdate', time=datetime.datetime(2020, 7, 11, 11, 27, 49, 805000), userId='abcdef..@thread.skype', chatId='19:abcdef..@thread.skype', content='<pictureupdate><eventtime>1594466869930</eventtime><initiator>8:username</initiator><value>URL@https://api.asm.skype.com/v1/objects/0-weu-d15-abcdef..</value></pictureupdate>')
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_PICTURE,
type=skypedata.MESSAGE_TYPE_TOPIC)
try:
tag = BeautifulSoup and BeautifulSoup(item.content, "html.parser").find("initiator")
if tag and tag.text:
author = id_to_identity(tag.text)
result.update(author=author)
result.update(from_dispname=self.get_contact_name(author))
except Exception:
logger.warning("Error parsing author from message %r.", item, exc_info=True)
elif "RichText/Media_Video" == item.type \
or "RichText/Media_AudioMsg" == item.type:
# SkypeVideoMsg(id='1594466637922', type='RichText/Media_Video', time=datetime.datetime(2020, 7, 11, 11, 23, 10, 45000), clientId='7459423203289210001', userId='username', chatId='8:username', content='<URIObject uri="https://api.asm.skype.com/v1/objects/0-weu-d8-abcdef.." url_thumbnail="https://api.asm.skype.com/v1/objects/0-weu-d8-abcdef../views/thumbnail" type="Video.1/Message.1" doc_id="0-weu-d8-abcdef.." width="640" height="480">To view this video message, go to: <a href="https://login.skype.com/login/sso?go=xmmfallback?vim=0-weu-d8-abcdef..">https://login.skype.com/login/sso?go=xmmfallback?vim=0-weu-d8-abcdef..</a><OriginalName v="937029c3-6a12-4202-bdaf-aac9e341c63d.mp4"></OriginalName><FileSize v="103470"></FileSize></URIObject>')
# For audio: type="Audio.1/Message.1", "To hear this voice message, go to: ", ../views/audio
result.update(type=skypedata.MESSAGE_TYPE_SHARE_VIDEO2)
elif isinstance(item, skpy.SkypeImageMsg):
# SkypeImageMsg(id='1594466401638', type='RichText/UriObject', time=datetime.datetime(2020, 7, 11, 11, 19, 13, 899000), clientId='566957100626477146', userId='username', chatId='8:username', content='<URIObject uri="https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef.." url_thumbnail="https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef../views/imgt1_anim" type="Picture.1" doc_id="0-weu-d3-abcdef.." width="1191" height="619">To view this shared photo, go to: <a href="https://login.skype.com/login/sso?go=xmmfallback?pic=0-weu-d3-abcdef..">https://login.skype.com/login/sso?go=xmmfallback?pic=0-weu-d3-abcdef..</a><OriginalName v="f.png"></OriginalName><FileSize v="108188"></FileSize><meta type="photo" originalName="f.png"></meta></URIObject>', file=File(name='f.png', size='108188', urlFull='https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef..', urlThumb='https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef../views/imgt1_anim', urlView='https://login.skype.com/login/sso?go=xmmfallback?pic=0-weu-d3-abcdef..'))
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_SPECIAL,
type=skypedata.MESSAGE_TYPE_SHARE_PHOTO)
elif isinstance(item, skpy.SkypeFileMsg): # Superclass for SkypeImageMsg, SkypeAudioMsg, SkypeVideoMsg
# SkypeFileMsg(id='1594466346559', type='RichText/Media_GenericFile', time=datetime.datetime(2020, 7, 11, 11, 18, 19, 39000), clientId='8167024171841589273', userId='username', chatId='8:username', content='<URIObject uri="https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef.." url_thumbnail="https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef../views/original" type="File.1" doc_id="0-weu-d3-abcdef..">To view this file, go to: <a href="https://login.skype.com/login/sso?go=webclient.xmm&docid=0-weu-d3-abcdef..">https://login.skype.com/login/sso?go=webclient.xmm&docid=0-weu-d3-abcdef..</a><OriginalName v="f.txt"></OriginalName><FileSize v="447"></FileSize></URIObject>', file=File(name='f.txt', size='447', urlFull='https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef..', urlThumb='https://api.asm.skype.com/v1/objects/0-weu-d3-abcdef../views/original', urlView='https://login.skype.com/login/sso?go=webclient.xmm&docid=0-weu-d3-abcdef..'))
filename, filesize = (item.file.name, item.file.size) if item.file else (None, None)
fileurl = item.file.urlFull
result.update(chatmsg_type=skypedata.CHATMSG_TYPE_SPECIAL, type=skypedata.MESSAGE_TYPE_FILE,
body_xml='<files><file index="0" size="%s" url="%s">%s</file></files>' %
(urllib.parse.quote(util.to_unicode(filesize or 0)),
urllib.parse.quote(util.to_unicode(fileurl or ""), safe=":/"),
util.to_unicode(filename or "file").replace("<", "<").replace(">", ">")))
else: # SkypeCardMsg, SkypeChangeMemberMsg, ..
result = None
return result
def populate(self, chats=()):
"""
Retrieves account information, all or selected chats with messages, and saves to database.
@param chats list of chat identities to populate if not everything
"""
if self.populated:
self.skype = None
self.login() # Re-login to reset skpy query cache
self.db.ensure_schema()
if not chats: self.populate_account()
self.populate_chats(chats)
self.populated = True
def populate_account(self):
"""Retrieves account profile data and saves to database."""
if not self.progress(action="populate", table="accounts", start=True): return
self.sync_counts.clear()
self.build_cache("accounts")
logger.info("Synchronizing database account profile from live.")
updateds = set()
try:
account = self.reqattr(self.skype, "user")
_, action = self.save("accounts", account)
if action in (self.SAVE.INSERT, self.SAVE.UPDATE):
updateds.add(self.db.id)
finally:
self.progress(action="populate", table="accounts", identities=list(updateds), end=True)
def populate_contacts(self, contacts=()):
"""
Retrieves contact profiles and saves to database.
@param contacts list of contact identities to populate if not all
"""
self.request(self.skype.contacts.sync) # Populate contacts in skpy
total = len(contacts or self.skype.contacts)
if not self.progress(action="populate", table="contacts", start=True):
return
getter = lambda x: self.request(self.skype.contacts.__getitem__, x, __raise=False)
makeid = lambda x: re.sub(r"^\d+\:", "", x) # Strip numeric prefix
iterable = iter((getter(makeid(x)) for x in contacts) if contacts else self.skype.contacts)
self.sync_counts.clear()
self.build_cache("contacts")
logger.info("Synchronizing %s from live.",
util.plural("contact profile", total, numbers=bool(contacts)))
updateds = set()
try:
for i, contact in enumerate(iterable):
if not i % 10 and not self.progress(
action="info", message="Querying contacts..",
**(dict(index=i + 1, count=total) if total > 1 else {})
): break # for i, contact
dbitem, action = self.save("contacts", contact)
if action in (self.SAVE.INSERT, self.SAVE.UPDATE):
updateds.add(dbitem["skypename"])
finally:
self.progress(action="populate", table="contacts", end=True,
count=self.sync_counts["contacts_updated"],
new=self.sync_counts["contacts_new"], identities=list(updateds))
def populate_chats(self, chats=(), messages=True):
"""
Retrieves all conversations and saves to database.
@param chats list of chat identities to populate if not all
@param messages whether to retrieve messages, or only chat metainfo and participants
"""
cstr = "%s " % util.plural("chat", chats, numbers=False) if chats else ""
logger.info("Starting to sync %s'%s' from Skype online service as '%s'.",
cstr, self.db, self.username)
if not self.progress(action="populate", table="chats", start=True):
return
self.build_cache()
self.sync_counts.clear()
def get_live_chats(identities, older=False):
"""Yields skpy.SkypeChat instances for identities"""
for i, k in enumerate(map(identity_to_id, identities or ())):
if not self.progress(
action="info",
message="Querying %s chats.." % ("older" if older else "selected"),
**(dict(index=i + 1, count=len(identities)) if len(identities) > 1 else {})
): break # for k
v = self.request(self.skype.chats.chat, k, __raise=False, __log=False)
if v: yield v
selchats = get_live_chats(chats)
processeds, updateds, completeds, msgids = set(), set(), set(), set()
run = True
while run:
if chats: mychats = selchats
elif not self.progress(action="info", message="Querying recent chats.."):
break # while run
else: mychats = self.request(self.skype.chats.recent, __raise=False).values()
for chat in mychats:
if not self.process_chat(chat, messages, processeds, updateds, completeds, msgids):
run = False
break # for chat
if chats: break # while run; stop after processing specific chats
elif run and not mychats \
and conf.Login.get(self.db.filename, {}).get("sync_older", True):
# Exhausted recents: check other existing chats as well
chats = set(self.cache["chats"]) - processeds
selchats = get_live_chats(chats, older=True)
if not chats: break # while run
elif not mychats: break # while run
pargs = dict(
message_count_new=self.sync_counts["messages_new"],
message_count_updated=self.sync_counts["messages_updated"],
) if messages else {}
self.progress(
action="populate", table="chats", end=True, count=len(updateds),
new=self.sync_counts["chats_new"], identities=list(updateds),
contact_count_new=self.sync_counts["contacts_new"],
contact_count_updated=self.sync_counts["contacts_updated"], **pargs
)
for dct in (self.cache, self.msg_lookups, self.msg_stamps): dct.clear()
logger.info("Finished syncing %s'%s'.", cstr, self.db)
def process_chat(self, chat, full, chats_processed, chats_updated, chats_completed, messages_processed):
"""
Retrieves chat metainfo and messages and saves to database.
@param chat skpy.SkypeChat instance
@param full retrieve messages, or only chat metainfo and participants
@param chats_processed state parameter: set of chat identities processed
@param chats_updated state parameter: set of chat identities updated
@param chats_completed state parameter: set of chat identities finished
@param messages_processed state parameter: set of message identities processed
@return whether further progress should continue
"""
result = True
cidentity = chat.id if isinstance(chat, skpy.SkypeGroupChat) \
or is_bot(self.reqattr(chat, "user")) \
or chat.id.startswith(skypedata.ID_PREFIX_BOT) else chat.userId
if chat.id.startswith(skypedata.ID_PREFIX_SPECIAL): # Skip specials like "48:calllogs"
return result
if isinstance(chat, skpy.SkypeGroupChat) and not self.reqattr(chat, "userIds"):
# Weird empty conversation, getMsgs raises 404
return result
if isinstance(chat, skpy.SkypeSingleChat) and chat.userId == self.skype.userId:
# Conversation with self?
return result
if cidentity in chats_processed: return result
chats_processed.add(cidentity)
if not self.progress(action="populate", table="chats", chat=cidentity, start=True):
return False
count, new, updated, run, start = 0, 0, 0, True, bool(full)
first, last = datetime.datetime.max, datetime.datetime.min
while run:
msgs, max_tries = [], 3 if start else 1
while full and max_tries and not msgs: # First or second call can return nothing
msgs = self.request(chat.getMsgs, __raise=False) or []
max_tries -= 1
if not full or msgs and (cidentity not in self.cache["chats"]
or (conf.Login.get(self.db.filename, {}).get("sync_contacts", True)
and not self.cache["chats"][cidentity].get("__updated__"))):
# Insert chat or update existing contacts, only if doing metainfo or if any messages
try: _, action = self.save("chats", chat)
except Exception:
logger.exception("Error saving chat %r.", chat)
else:
if action in (self.SAVE.INSERT, self.SAVE.UPDATE):
chats_updated.add(cidentity)
if not self.progress(action="populate", table="chats", chat=cidentity):
break # while run
if start: self.build_msg_cache(cidentity)
if not self.progress(action="populate", table="messages", chat=cidentity,
count=count, new=new, updated=updated, start=start):
result = run = False
break # while run
start = False
for msg in msgs:
try: _, action = self.save("messages", msg, parent=chat)
except Exception:
logger.exception("Error saving message %r.", msg)
if not self.progress():
msgs, run = [], False
break # for msg
continue # for msg
if self.SAVE.SKIP != action: count += 1
if self.SAVE.INSERT == action: new += 1
if self.SAVE.UPDATE == action: updated += 1
if action in (self.SAVE.INSERT, self.SAVE.UPDATE):
first, last = min(first, msg.time), max(last, msg.time)
if not self.progress(action="populate", table="messages", chat=cidentity,
count=count, new=new, updated=updated):
msgs, run = [], False
break # for msg
if self.SAVE.NOCHANGE == action and msg.id not in messages_processed:
msgs = [] # Stop on reaching already retrieved messages
break # for msg
messages_processed.add(msg.id)
if not msgs:
break # while run
if (new or updated):
chats_updated.add(cidentity)
if cidentity in chats_updated and cidentity in self.cache["chats"] and self.db.is_open():
row = self.db.execute("SELECT id, convo_id, MIN(timestamp) AS first, "
"MAX(timestamp) AS last FROM messages WHERE convo_id = :id",
self.cache["chats"][cidentity]
).fetchone()
self.db.execute("UPDATE conversations SET last_message_id = :id, "
"last_activity_timestamp = :last, "
"creation_timestamp = COALESCE(creation_timestamp, :first) "
"WHERE id = :convo_id", row)
chats_completed.add(cidentity)
pargs = dict(action="populate", table="chats", chat=cidentity)
if full: pargs.update(table="messages", count=count, new=new, updated=updated,
first=first, last=last, end=True)
if not self.progress(**pargs):
result = False
return result
def get_api_content(self, url, category=None, cache=True):
"""
Returns content raw binary from Skype API URL via login, or None.
@param category type of content, e.g. "avatar" for avatar image,
"file" for shared file
@param cache whether to use disk cache if media caching is enabled;
loads content from disk if available, and saves download to disk
"""
result, cached, url = None, None, make_content_url(url, category)
urls = [url]
# Some images appear to be available on one domain, some on another
if not url.startswith("https://experimental-api.asm"):
url0 = re.sub(r"https\:\/\/.+\.asm", "https://experimental-api.asm", url)
urls.insert(0, url0)
for url in urls if conf.SharedContentUseCache and cache else ():
try:
filebase = os.path.join(self.dl_cache_dir, urllib.parse.quote(url, safe=""))
for p in glob.glob(filebase + ".*")[:1]:
with open(p, "rb") as f:
result = cached = f.read()
except Exception: pass
for url in urls if not result else ():
result = self.download_content(url, category)
if result: break # for url
if result and conf.SharedContentUseCache and cache and not cached:
try:
filetype = util.get_file_type(result, category, url)
filename = "%s.%s" % (urllib.parse.quote(url, safe=""), filetype)
filepath = os.path.join(self.dl_cache_dir, filename)
self.ensure_cache_dir()