-
Notifications
You must be signed in to change notification settings - Fork 15
/
scratchapi_new.py
1151 lines (937 loc) · 48.5 KB
/
scratchapi_new.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Copyright (c) 2015 dyspore.cc
The MIT License (MIT)
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files
(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
# ScratchAPI 2.0
# Written by Dylan5797/PolyEdge [http://dyspore.cc]
# __
# ____/ /_ ___________ ____ ________ __________
# / __ / / / / ___/ __ \/ __ \/ ___/ _ \ / ___/ ___/
# / /_/ / /_/ (__ ) /_/ / /_/ / / / __// /__/ /__
# \__,_/\__, /____/ .___/\____/_/ \___(_)___/\___/
# /____/ /_/
import traceback as _traceback
import requests as _requests
import json as _json
import hashlib as _hashlib
import os as _os
import re as _re
import time as _time
import webbrowser as _web
import asyncio as _asyncio
import websockets as _websockets
import websockets.client as _websockets_client
import warnings as _warnings
import io as _io
import zipfile as _zipfile
import urllib.parse as _urllib_parse
# EXCEPTIONS
class ScratchAPIExceptionBase(BaseException): "Base exception for the scratchapi module"
class Unauthenticated(ScratchAPIExceptionBase): "Raised when an authenticated action is attempted without logging in"
class InvalidCSRF(ScratchAPIExceptionBase): "Raised when the CSRF token is invalid or not fetched"
# UTIL EXTENDS
class _ScratchUtils:
def _tree(self, item, *args):
"reads a value from a data structure using the given tree"
path = args[:-1]
default = args[-1]
for x in path:
try:
item = item[x]
except (KeyError, IndexError):
return default
return item
def _ptree(self, value, item, *args, force=False):
"returns the `value` passed if not None, otherwise self._tree(item, *args)"
if (value is not None) and not force:
return value
return self._tree(item, *args)
def _put_nn(self, data, key, value):
if value is not None:
data[key] = value
# CLIENT SESSION
class ScratchSession:
def __init__(self, username=None, password=None, auto_login=True, retain_password=False):
"""Creates a scratch session. By default when a username and password are passed, the session automatically logs in.
Setting retain_password to true will allow login() to be called later if necessary without needing the password"""
self.SERVER = 'scratch.mit.edu'
self.API_SERVER = 'api.scratch.mit.edu'
self.PROJECTS_SERVER = 'projects.scratch.mit.edu'
self.ASSETS_SERVER = 'assets.scratch.mit.edu'
self.CDN_SERVER = 'cdn.scratch.mit.edu'
self.CLOUD = 'clouddata.scratch.mit.edu'
self.username = username
self.password = None
self.retain_password = retain_password
self.http_session = _requests.session()
self._save(username, password)
if username is not None and password is not None and auto_login:
self.login(username, password)
def _save(self, username, password):
self.username = username
if self.retain_password:
self.password = password
def get_csrf(self):
"Gets the CSRF token currently in use"
return self.http_session.cookies.get('scratchcsrftoken')
def get_session_id(self):
"Gets the session ID currently in use"
return self.http_session.cookies.get('scratchsessionsid')
def _cookie_header(
self): # ironically, CookieJar does not supply a direct method of fetching a cookie header, here is a (probably not up to spec) method to do so.
cookies = []
for name, value in self.http_session.cookies.get_dict().items():
cookies.append(name + "=" + value)
return "; ".join(cookies)
def _get_headers(self, cookie=False):
headers = {
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'https://scratch.mit.edu/',
'Origin': 'https://scratch.mit.edu'
}
if self.get_csrf() is not None:
headers["X-CSRFToken"] = self.get_csrf()
if cookie:
headers["Cookie"] = self._cookie_header()
return headers
def authenticated(self):
"Returns true if the session is authenticated, false otherwise"
return self.get_session_id() is not None
def rfa(self):
"Raise-for-authentication. Raises an error if the session is not authenticated"
_assert(self.authenticated(), Unauthenticated())
def new_csrf(self):
"Fetches a new CSRF token"
self.http('/csrf_token/', auth=False)
_assert(self.get_csrf() is not None, InvalidCSRF("the csrf token could not be fetched"))
return self.get_csrf()
def csrf(self):
"Fetches a new CSRF token if needed"
if self.get_csrf() is None:
self.new_csrf()
def login(self, *args):
"""Logs into scratch. If a username is supplied at class construction, a username does not need to be passed to login().
If retain_password is set at class construction, no arguments need to be passed"""
username = self.username
password = self.password
if len(args) == 0:
assert username is not None and password is not None
elif len(args) == 1:
assert username is not None
password = args[0]
elif len(args) == 2:
username = args[0]
password = args[1]
else:
raise ValueError("wrong number of args")
self.csrf()
self._save(username, password)
return self._login(username, password)
def _login(self, username, password):
return self.http('POST /login/',
body={'username': username, 'password': password, 'csrftoken': self.get_csrf()}, auth=False)
def logout(self):
"Logs out of scratch. Does not clear CSRF token"
return self.http('POST /accounts/logout/')
def purge(self):
"Clears all session data but DOES NOT log out. Use purge() only after logout()"
self.http_session.cookies.clear()
def session_valid(self):
"Checks whether the session is valid. May raise an error if scratch is down or the internet connection is lost."
return self.http('/messages/ajax/get-message-count/').status_code == 200
def ahttp(self, *args, **kwargs):
"Alias of http() with auth set to true"
return self.http(*args, **kwargs, auth=True)
def http(self, *args, **kwargs):
"""Makes a HTTP request. The request may be supplied in keyword arg only form, but can also be supplied using a specific argument pattern.
Pattern syntax: http([SERVER], "[METHOD] PATH", [tuple(FIELDS)], PAYLOAD, **kwargs)
> SERVER: optional server, defaults to self.SERVER
> "METHOD PATH": a string containing an optional request method and the request path. Any text following a colon (:) will be treated as a field,
similar to python %s syntax. When fields are specified, you must include a TUPLE, LIST, OR DICT of the fields directly after the path.
> PAYLOAD: The request body. If not bytes, will default to UTF-8 encoding. You may pass a JSON serializable object and it will be converted to JSON internally.
NOTE: eyword arguments will overwrite anything defined using the argument pattern defined above.
Keyword args:
> method: sets the request method
> server: sets the server being requested to. Defaults to self.SERVER
> body: The request body. Follows same behavior as PAYLOAD above
> payload: alias of body
> headers: dictionary of additional request headers to send to the server.
> retry: number of retries on request failure (connection errors etc). Defaults to 3.
> protocol: sets the request protocol. Defaults to https. Must be supported by the requests module.
> port: sets the request port. The default port defined by requests will be used when none is passed.
> auth: whether the request must be authenticated. Defaults to true. When set to true, an error will be raised if the session is not authenticated.
> rfs: defaults to true. When true, calls request.raise_for_status() when completed.
"""
request_protocol = "https"
request_server = self.SERVER
request_port = None
request_method = "GET"
request_path = None
request_authenticated = False
request_headers = {}
request_retries = 3
request_body = None
request_response = None
request_raise_for_status = True
if len(args) > 0:
a0_split = args[0].split(" ", 1)
if args[0].startswith('/'):
request_path = args[0]
args = args[1:]
elif len(a0_split) >= 2 and a0_split[1].startswith("/"):
request_method = a0_split[0].upper()
request_path = a0_split[1]
args = args[1:]
else:
request_server = args[0]
if args[1].startswith('/'):
request_path = args[1]
else:
a1_split = args[1].split(" ", 1)
request_method = a1_split[0].upper()
request_path = a1_split[1]
args = args[2:]
if (not "field" in kwargs) or kwargs["field"]:
path_build = ""
index = 0
field_arg = None
for x in _re.split("(:[a-zA-Z0-9]+)", request_path):
if x.startswith(":"):
if field_arg is None:
assert len(
args) > 0, "when supplying url fields using /:field/, supply a tuple, list or dictionary after the path or include (field=False) to disable field checking and use the raw URL."
field_arg = args[0] if type(args[0]) in [tuple, list, dict] else (args[0],)
args = args[1:]
if type(field_arg) == dict:
# noinspection PyTypeChecker
path_build += field_arg[x[1:]]
else:
path_build += field_arg[index]
index += 1
else:
path_build += x
request_path = path_build
if len(args) != 0:
request_body = args[0]
# args = args[1:]
if 'method' in kwargs:
request_method = kwargs['method'].upper()
if 'server' in kwargs:
request_server = kwargs['server']
if 'body' in kwargs:
request_body = kwargs['body']
if 'payload' in kwargs: # alias of body
request_body = kwargs['payload']
if 'headers' in kwargs:
request_headers.update(kwargs['headers'])
if 'retry' in kwargs:
request_retries = kwargs['retry']
if 'protocol' in kwargs:
request_protocol = kwargs['protocol']
if 'port' in kwargs:
request_port = kwargs['port']
if 'auth' in kwargs:
request_authenticated = kwargs['auth']
if 'rfs' in kwargs:
request_raise_for_status = kwargs['rfs']
request_headers.update(self._get_headers())
if request_authenticated:
_assert(self.get_session_id() is not None, Unauthenticated())
_assert("X-CSRFToken" in request_headers, InvalidCSRF())
_assert(request_headers["X-CSRFToken"] is not None, InvalidCSRF())
if not ((type(request_body) == bytes) or request_body is None):
if type(request_body) in [list, dict]:
request_body = _json.dumps(request_body).encode("utf-8")
elif type(request_body) == str:
request_body = request_body.encode("utf-8")
else:
raise TypeError("strange request body. expected bytes, str or json serializable object")
if request_body is not None:
request_headers["content-length"] = len(request_body)
request_headers = {
x: (str(request_headers[x]) if not isinstance(request_headers[x], (str, bytes)) else request_headers[x]) for
x in request_headers}
request_url = request_protocol + "://" + request_server + (
":" + str(request_port) if request_port is not None else "") + request_path
request_unprepared = _requests.Request(method=request_method.upper(), url=request_url, headers=request_headers,
cookies=self.http_session.cookies)
if request_body is not None:
request_unprepared.data = request_body
request_prepared = request_unprepared.prepare()
for x in range(0, request_retries + 1):
if x >= request_retries:
raise ConnectionError('Connection failed on all ' + str(request_retries) + ' attempts')
try:
request_response = self.http_session.send(request_prepared)
break
except _requests.exceptions.BaseHTTPError:
continue
if request_raise_for_status and request_response is not None:
request_response.raise_for_status()
return request_response
# AUTH HELPER
class _ScratchAuthenticatable:
""""Represents an online scratch entity that may contain additional features when the client is authenticated.
An instance of ScratchSession may be stored in an instance of ScratchAuthenticatable in order for it to function without requiring a session to be passed every time"""
def _chain(self, instance):
"used by a parent authenticatable to inherit this authenticatables session from its session"
assert instance != self
self._scratch_session = instance._session()
return self
def _put_auth(self, session):
"used by a parent authenticatable to set the session of this authenticatable"
self._scratch_session = session
return self
def _auth(self, session: ScratchSession = None) -> ScratchSession():
"alias of _session(session, auth=True)"
return self._session(session=session, auth=True)
def _session(self, session: ScratchSession = None, auth=False) -> ScratchSession():
"""returns the session the authenticatable is using unless another session is specified in the arguments as an override. will make a new session if needed.
if auth is set to true and the session is not authenticated, will raise an error"""
if getattr(self, "_scratch_session", None) is None:
self._scratch_session = None
if session is None:
if self._scratch_session is None:
self._scratch_session = ScratchSession()
session = self._scratch_session
if auth:
_assert(session.authenticated(), Unauthenticated())
session.csrf()
return session
def authenticate(self, session: ScratchSession):
"authenticates the object with the given session."
self._scratch_session = session
class _ScratchDSObjectMapping:
"Represents a data structure provided by the scratch website that can be directly be converted to an object and back"
def __init__(self):
self.from_mapping = {}
self.to_mapping = {}
def fm(self, ds_name, ob_name):
self.from_mapping[ds_name] = ob_name
def to(self, ob_name, ds_name):
self.to_mapping[ob_name] = ds_name
def bt(self, ob_name, ds_name):
"object name first"
self.to(ob_name, ds_name)
self.fm(ds_name, ob_name)
def convert_from(self, ds, obj, extra=True):
for x in ds:
if x in self.from_mapping:
setattr(obj, self.from_mapping[x], ds[x])
elif extra:
if not hasattr(obj, "ds_extra"):
setattr(obj, "ds_extra", {})
getattr(obj, "ds_extra")[x] = ds[x]
def convert_to(self, obj, lenient=True, ignore_none=True):
ds = {}
for x in self.to_mapping:
if hasattr(obj, x):
if (getattr(obj, x) is None) and ignore_none:
continue
ds[self.to_mapping[x]] = getattr(obj, x)
elif not lenient:
raise AttributeError("object %s does not have attribute %s" % (obj, x))
return ds
class ScratchAPI:
"NOT UPDATED YET! A wrapper"
def __init__(self, *args, **kwargs):
auto_csrf = True
if "auto_csrf" in kwargs:
auto_csrf = kwargs["auto_csrf"]
del kwargs["auto_csrf"]
if len(args) >= 1 and type(args[0]) == ScratchSession:
self.session = args[0]
else:
self.session = ScratchSession(*args, **kwargs)
if auto_csrf:
self.session.csrf()
def projects_legacy_get(self, project_id):
return self.session.http(path='/internalapi/project/' + project_id + '/get/',
server=self.session.PROJECTS_SERVER).json()
def projects_legacy_set(self, project_id, payload):
return self.session.http(server=self.session.PROJECTS_SERVER,
path='/internalapi/project/' + project_id + '/set/', payload=payload, method='post')
def projects_get_meta(self, project_id):
return self.session.http(path='/api/v1/project/' + str(project_id) + '/?format=json').json()
def projects_get_remixtree(self, project_id):
return self.session.http(path='/projects/' + str(project_id) + '/remixtree/bare/').json()
def _tools_verifySession(self):
return self.session.http(path='/messages/ajax/get-message-count/', port=None).status_code == 200
def _backpack_getBackpack(self):
return self.session.http(path='/internalapi/backpack/' + self.lib.set.username + '/get/').json()
def _backpack_setBackpack(self, payload):
return self.session.http(server=self.CDN_SERVER,
path='/internalapi/backpack/' + self.lib.set.username + '/set/', method="post",
payload=payload)
def _userpage_setStatus(self, payload):
p2 = self.session.http(path='/site-api/users/all/' + self.lib.set.username).json()
p = {}
for i in p2:
if i in ['comments_allowed', 'id', 'status', 'thumbnail_url', 'userId', 'username']:
p[i] = p2[i]
p['status'] = payload
return self.session.http(path='/site-api/users/all/' + self.lib.set.username, method="put",
payload=_json.dumps(p))
def _userpage_toggleComments(self):
return self.session.http(path='/site-api/comments/user/' + self.lib.set.username + '/toggle-comments/',
method="put")
def _userpage_setBio(self, payload):
p2 = self.session.http(path='/site-api/users/all/' + self.lib.set.username).json()
p = {}
for i in p2:
if i in ['comments_allowed', 'id', 'bio', 'thumbnail_url', 'userId', 'username']:
p[i] = p2[i]
p['bio'] = payload
return self.session.http(path='/site-api/users/all/' + self.lib.set.username, method="put",
payload=_json.dumps(p))
def _users_get_meta(self, usr):
return self.session.http(path='/users/' + usr, server=self.API_SERVER).json()
def _users_follow(self, usr):
return self.session.http(path='/site-api/users/followers/' + usr + '/add/?usernames=' + self.lib.set.username,
method='PUT')
def _users_unfollow(self, usr):
return self.session.http(
path='/site-api/users/followers/' + usr + '/remove/?usernames=' + self.lib.set.username, method='PUT')
def _users_comment(self, user, comment):
return self.session.http(path='/site-api/comments/user/' + user + '/add/', method='POST',
payload=_json.dumps({"content": comment, "parent_id": '', "commentee_id": ''}))
def _studios_comment(self, studioid, comment):
return self.session.http(path='/site-api/comments/gallery/' + str(studioid) + '/add/', method='POST',
payload=_json.dumps({"content": comment, "parent_id": '', "commentee_id": ''}))
def _studios_get_meta(self, studioid):
return self.session.http(path='/site-api/galleries/all/' + str(studioid)).json()
def _studios_invite(self, studioid, user):
return self.session.http(
path='/site-api/users/curators-in/' + str(studioid) + '/invite_curator/?usernames=' + user, method='PUT')
def _projects_comment(self, project_id, comment):
return self.session.http(path='/site-api/comments/project/' + str(project_id) + '/add/', method='POST',
payload=_json.dumps({"content": comment, "parent_id": '', "commentee_id": ''}))
def _assets_get(self, md5):
return self.session.http(path='/internalapi/asset/' + md5 + '/get/', server=self.ASSETS_SERVER).content
def _assets_set(self, md5, content, content_type=None):
if not content_type:
if _os.path.splitext(md5)[-1] == '.png':
content_type = 'image/png'
elif _os.path.splitext(md5)[-1] == '.svg':
content_type = 'image/svg+xml'
elif _os.path.splitext(md5)[-1] == '.wav':
content_type = 'audio/wav'
else:
content_type = 'text/plain'
headers = {'Content-Length': str(len(content)),
'Origin': 'https://cdn.scratch.mit.edu',
'Content-Type': content_type,
'Referer': 'https://cdn.scratch.mit.edu/scratchr2/static/__cc77646ad8a4b266f015616addd66756__/Scratch.swf'}
return self.session.http(path='/internalapi/asset/' + md5 + '/set/', method='POST', server=self.ASSETS_SERVER,
payload=content)
def _users_get_message_count(self, user=None):
if user == None:
user = self.lib.set.username
return self.session.http(path='/proxy/users/' + user + '/activity/count', server=self.API_SERVER).json()[
'msg_count']
def _get_message_html(self):
return self.session.http(path='/messages/')
class ScratchUserSession:
def __init__(self, *args, **kwargs):
_warnings.warn("""Scratch user sessions and interfacing with the API have been seperated into 2 classes, ScratchSession and ScratchAPI.
ScratchUserSession has been made an alias of ScratchAPI, which will still accept a username and password to create its own ScratchSession with.
This alias may be removed in the future, it is suggested that you change over to the new class names.""")
super().__init__(*args, **kwargs)
# OFFLINE PROJECTS
class ScratchProject:
def __init__(self, load=None, reader=None):
"""represents an offline scratch project. In order to upload a scratch project, you must create an online project first
(using ScratchAPI.new_project() or creating a new OnlineScratchProject)"""
self.json = None
self.assets = {}
self.reader = reader
self.stream = None
if load is not None:
ScratchProject.load(load, reader)
def load(self, stream, reader=None):
project_json, stream = self._stream(stream)
if reader is None:
reader = _ScratchProjectReader.get_reader(project_json)
assert reader is not None, "couldn't read this file. no reader accepted the JSON content."
self.json = project_json
self.reader = reader
self.stream = stream
self._read_assets(self.json, self.stream)
def _get_zipfile(self, stream):
if type(stream) == bytes:
stream = _io.BytesIO(stream)
return _zipfile.ZipFile(stream)
def _stream(self, stream):
try:
scratch_file = self._get_zipfile(stream)
except:
raise ValueError("scratch file is invalid. NOTE: scratch 1.4 is not supported")
json_filename = ([x.filename for x in scratch_file.filelist if x.filename == 'project.json'] + [x.filename for x in scratch_file.filelist if 'project.json' in x.filename]) # have seen prefixed project.json files that scratch still opens ...lol
json_file = scratch_file.open(json_filename)
project_json = _json.loads(json_file.read().decode('utf-8'))
json_file.close()
return project_json, scratch_file
def _read_assets(self, json, stream):
return self.reader.from_binary(json, stream, self)
def get_json(self):
"gets the JSON of the project"
assert self.json is not None
return self.json
def get_asset(self, name):
"gets an asset with the specified name, returns None if not found."
if not (name in self.assets):
self.read_asset(name)
if name in self.assets:
return self.assets[name]
return None
def put_asset(self, data: bytes, filetype):
"adds an asset to the project"
md5 = _hashlib.md5(data).hexdigest()
self.assets[md5 + "." + filetype] = data
return md5
def del_asset(self, filename):
"removes an asset with the specified name"
while filename in self.assets:
del self.assets[filename]
# Override methods
def read_asset(self, name):
"will return an asset with the name specified"
if name in self.assets:
return self.assets[name]
return None
class _ScratchProjectReader(_ScratchUtils):
def __init__(self, parent):
self.parent = parent
def get_reader(*args):
parent = args[-1]
for reader in _ScratchProjectReader.__subclasses__():
instance = reader(parent)
if instance.includes(args[-1]):
return instance
return None
def includes(self, json):
"returns true if the passed json is implemented by this reader"
raise NotImplemented()
def from_binary(self, json, stream, parent: ScratchProject):
"generates a list of all assets from a scratch binary stream"
raise NotImplemented()
def to_binary(self, json, assets):
"converts json and assets to a scratch binary file"
raise NotImplemented()
def get_asset_list(self, json): # unused for now
"creates an asset list from the passed json object"
raise NotImplemented()
class _ScratchReader20(_ScratchProjectReader):
def includes(self, json):
raise NotImplemented("not currently implemented.")
def from_binary(self, json, stream, parent):
raise NotImplemented("not currently implemented.")
def to_binary(self, json, assets):
raise NotImplemented("not currently implemented.")
def get_asset_list(self, json):
raise NotImplemented("not currently implemented.")
class _ScratchReader30(_ScratchProjectReader):
def includes(self, json):
raise NotImplemented("not currently implemented.")
def from_binary(self, json, stream, parent):
raise NotImplemented("not currently implemented.")
def to_binary(self, json, assets):
raise NotImplemented("not currently implemented.")
def get_asset_list(self, json):
raise NotImplemented("not currently implemented.")
# ONLINE PROJECTS
class ScratchProjectOnlineProxy(ScratchProject, _ScratchAuthenticatable):
def __init__(self, project_id=None):
"""a proxy implementation of ScratchProject representing a scratch project stored on MIT servers
NOTE: assets will not be re-uploaded on save unless they are downloaded. This API does not provide high level
project modification capabilities. If you wish to change the project assets, you must call project.put_asset()
with any assets that you are adding to project, unless you know for sure they exist on the scratch assets server."""
self.project_id = None
super().__init__(self, load=project_id, instance=self)
def _get_zipfile(self, stream):
raise NotImplemented("zipfile loading is not supported in this implementation")
def _stream(self, stream):
assert type(stream) in [int, str], "expected a project ID but received " + type(stream).__name__
stream = str(stream)
self.project_id = stream
session = self._session()
project_json = session.http(session.PROJECTS_SERVER, "/:id", stream).json()
return project_json, stream
def _read_assets(self, json, stream):
return None
class OnlineScratchProject(_ScratchAuthenticatable, _ScratchUtils):
def __init__(self, project_id=None):
"Represents a project stored on the scratch website. May not exist on the website yet if id is none"
self.project_id = project_id
self.loaded = False
self.author = None
self.title = None
self.shared = None
self.comments_enabled = None
self.created = None
self.modified = None
self.shared = None
self.thumbnails = None
self.instructions = None
self.credits = None
self.remix = None
self.remix_original = None
self.remix_parent = None
self.view_count = None
self.love_count = None
self.favorite_count = None
self.comment_count = None
self.remix_count = None
self.project = None
def _ensure_created(self):
assert self.created(), "the project must be created to execute this operation"
def _ensure_loaded(self, session=None):
if not self.loaded:
return self.load(session)
def _downstream(self, data, force=True):
# self.author = ??
self.shared = self._ptree(self.shared, data, "is_published", False, force=force)
self.comments_enabled = self._ptree(self.comments_enabled, data, "comments_allowed", False, force=force)
self.shared = self._ptree(data, "is_published", False, force=force)
def _upstream(self):
data = {}
self._put_nn(data, "title", self.title)
self._put_nn(data, "comments_allowed", self.comments_enabled)
self._put_nn(data, "instructions", self.instructions)
self._put_nn(data, "credits", self.credits)
def created(self):
"returns true if the project has an ID on the scratch website"
return self.project_id is not None
def new(self, session=None, project: ScratchProject=None, title=None, remix_parent=None, load=True):
"creates a new scratch project and saves it to this instance. If this instance is already a scratch project, raises an error."
assert not self.created(), "cannot overwrite an existing project with a new instance. if you wish to update the project's contents, "
session = self._auth(session)
project_json = {}
if project is not None:
project_json = project.get_json()
self._upload_assets(project)
qs = {}
if title is not None:
qs["title"] = title
if remix_parent is not None:
qs["is_remix"] = 1
qs["original_id"] = remix_parent
result = session.ahttp(session.PROJECTS_SERVER, "POST /:qs", _urllib_parse.urlencode(qs), payload=project_json).json()
assert result["status"] == "ok", 'server responded with unexpected status: "%s"' % result["status"]
self.project_id = result["content-name"]
if load:
self.load(session=session)
return self.project_id
def load(self, session=None):
"downloads the projects metadata from the scratch website"
self._ensure_created()
session = self._session(session)
self._downstream(session.http(session.API_SERVER, "/projects/:project", self.project_id).json())
def get_project(self, session=None):
"gets a ScratchProject-like object representing the project stored on scratch."
self._ensure_created()
project = ScratchProjectOnlineProxy(self.project_id)
project._put_auth(self._session(session))
return project
def _upload_assets(self, project, session=None):
pass # TODO
# CLOUD SESSIONS
class _CloudSessionWatchdog:
def __init__(self, instance):
self.enabled = True
self.instance = instance
self.read_last = _time.time()
self.read_timeout = 45 # after this many seconds with no reads, the connection will be reestablished
def reconnect_needed(self):
return _time.time() > self.read_last + self.read_timeout
def reset_read(self):
self.read_last = _time.time()
async def handle_connection(self):
if (not self.enabled) or not self.instance._connected:
return
if (not (self.instance.socket.state in [_websockets_client.OPEN, _websockets_client.CONNECTING])) or self.reconnect_needed():
self.instance._debug("watchdog: reconnect -> " + str(self.instance.socket.state))
self.reset_read()
await AIOCloudSession.reconnect(self.instance)
class AIOCloudSession: # not sure exactly the usage cases of this thing because the main api isn't AIO but better to include this than not
def __init__(self, *args, loop=None, **kwargs):
"""Creates an asyncio based cloud session. The event loop used defaults to asyncio.get_event_loop() unless one is passed with (loop=)
Passing a project ID as the first argument will set the project_id parameter in the created instance and will be connected to using connect()
A session must be provided in the form of a ScratchSession, ScratchAPI or username and password."""
self.event_loop = loop or _asyncio.get_event_loop()
self.project_id = None
self.session = None
if len(args) > 0 and (type(args[0]) == int or (len(args) == 3 and type(args[0]) == str)):
self.project_id = str(args[0])
args = args[1:]
if len(args) > 0:
if type(args[0]) == ScratchSession:
self.session = args[0]
elif type(args[0]) == ScratchAPI:
self.session = args[0].session
elif type(args[0]) in [list, tuple]:
self.session = ScratchSession(args[0][0], args[0][1], **kwargs)
else:
self.session = ScratchSession(*args, **kwargs)
self.watchdog = _CloudSessionWatchdog(self)
self._connected = False # whether the client should be connected. could be incorrect as to whether the socket is actually connected or not. use connected()
self._debug_enabled = False # displays diagnostics when set to true
self.variables = _AIOCloudVariablesList(self)
self._client_outbound = []
self._client_inbound = []
self.socket = None
# self._rollover = []
def _debug(self, message):
if self._debug_enabled:
print(message)
def _check_connected(self):
assert self._connected, "The client is not connected"
assert self.socket is not None, "No connection was established"
async def connect(self, project_id=None):
"""connects to the cloud data server. specifying a project_id will overwrite any id given when the class is constructed.
one must be given at class contruction or passed to connect() or an error will be raised"""
self.project_id = project_id or self.project_id
assert self.project_id is not None
assert self.session.authenticated()
_websockets_client.USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" # look like a normal client if this works the way i expect it does :sunglasses:
self._debug("connect: establish")
self.socket = await _websockets_client.connect("ws://" + self.session.CLOUD + "/", loop=self.event_loop,
extra_headers=[(key, value) for key, value in
self.session._get_headers(cookie=True).items()])
self.socket.timeout = 30 # not sure how big this is server side but this number will be used to mark the socket as timed out after a small period of no reads
self._connected = True
self.watchdog.reset_read()
self._debug("connect: handshake")
await AIOCloudSession._write_packet(self, self._create_packet('handshake', {}))
self._debug("connect: write()")
await AIOCloudSession._write(self)
self._debug("connect: complete")
async def disconnect(self, timeout=0.25):
"disconnects from the cloud data server"
try:
await _asyncio.wait_for(self.socket.close(), timeout)
except _asyncio.TimeoutError: # dirty disconnect but oh well i guess
del self.socket
self.socket = None
self._connected = False
async def reconnect(self):
"reconnects the client to the cloud data server, if connected, otherwise an error is raised"
self._check_connected()
self._debug("reconnect: disconnect")
await AIOCloudSession.disconnect(self)
self._debug("reconnect: connect")
await AIOCloudSession.connect(self)
async def keep_alive(self):
"keeps the connection alive if connected"
await AIOCloudSession._read(self, 0)
def _create_packet(self, method, options=None):
"creates a cloud data packet with the set method and options and parameters"
base_packet = {
'user': self.session.username,
'project_id': str(self.project_id),
'method': method
}
packet = (options or {}).copy()
packet.update(base_packet)
return packet
async def _read(self, timeout=0, max_count=None):
"reads updates from the websocket and keeps the connection alive"
AIOCloudSession._check_connected(self)
await self.watchdog.handle_connection()
max_count = float('inf') if max_count is None else max_count - 1
updates = []
count = -1
while count < max_count:
count += 1
try:
await _asyncio.sleep(0)
data = await _asyncio.wait_for(self.socket.recv(), timeout)
except _asyncio.TimeoutError:
break
except _websockets.ConnectionClosed:
await AIOCloudSession.reconnect(self)
continue
if (data is None) or (data == b""):
break
try:
packet = _json.loads(data)
except _json.JSONDecodeError:
_traceback.print_exc()
continue
updates.append(packet)
self._client_inbound.append(packet)
if ("method" in packet) and (packet["method"].lower().strip() == "set"):
self.variables._put(packet["name"], packet["value"])
self.watchdog.reset_read()
return updates
async def _write(self):
"sends all queued packets to the cloud data server"
self._debug("write: keep alive")
await AIOCloudSession.keep_alive(self)
self._debug("write: write")
for packet in self._client_outbound.copy():
del self._client_outbound[0]
try:
await AIOCloudSession._write_packet(self, packet)
except _websockets.ConnectionClosed:
self._debug("write: closed and reconnecting")
self._client_outbound.insert(0, packet)
await AIOCloudSession.reconnect(self)
return
self._debug("write: success")
self._debug("write: complete")
async def _write_packet(self, packet):
"sends a packet to the cloud data server"
ob = (_json.dumps(packet) + '\n').encode('utf-8')
self._debug("write_packet: sending " + str(ob))
await self.socket.send(ob)
async def _send(self, *args, **kwargs):
"queues a packet to be sent to the cloud data server. in a normal situation, the packet will be sent immediately"
packet = self._create_packet(*args, **kwargs)
self._debug("send: queueing " + str(packet))
self._client_outbound.append(packet)
self._debug("send: write()")
await AIOCloudSession._write(self)
self._debug("send: complete")
async def get_updates(self):
"fetches a list of packets received from the server"
await AIOCloudSession._read(self, 0)
updates = self._client_inbound.copy()
self._client_inbound.clear()
return updates
async def set_var(self, name, value):
"sets an existing cloud variable. a cloud symbol and space (☁ ) must be included in the name if present on the server"
await AIOCloudSession._send(self, 'set', {'name': name, 'value': value})
self.variables._put(name, value)
async def create_var(self, name, value=None):
"creates a cloud variable. a cloud symbol and space (☁ ) must be included in the name if present on the server"
value = value or 0
await AIOCloudSession._send(self, 'create', {'name': name, 'value': value})
async def rename_var(self, old_name, new_name):
"changes the name of an existing cloud variable. a cloud symbol and space (☁ ) must be included in both names if present on the server"
await AIOCloudSession._send(self, 'rename', {'name': old_name, 'new_name': new_name})
async def delete_var(self, name):
"deletes an existing cloud variable. a cloud symbol and space (☁ ) must be included in the name if present on the server"
await AIOCloudSession._send(self, 'delete', {'name': name})
async def get_var(self, name, timeout=0):
"""[it is recommended to use session.variables.get(name, default) instead]
gets the value a cloud variable, raises KeyError if not found. a cloud symbol and space (☁ ) must be included in the name if present on the server"""