/
api.py
1408 lines (1282 loc) · 73 KB
/
api.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import asyncio
import json
import logging
import os
import shutil
import string
import tempfile
import uuid
import zipfile
from asyncio import Future, Queue
from contextlib import asynccontextmanager
from datetime import timedelta, datetime, timezone
from functools import partial
from io import BytesIO
from pathlib import Path
from random import SystemRandom
from typing import (
AsyncGenerator,
Any,
Optional,
Sequence,
Union,
List,
Dict,
AsyncIterator,
Tuple,
Callable,
Awaitable,
Literal,
cast,
)
from urllib.parse import urlencode, urlparse, parse_qs, urlunparse
import aiohttp_jinja2
import jinja2
import prometheus_client
import yaml
from aiohttp import (
web,
MultipartWriter,
AsyncIterablePayload,
BufferedReaderPayload,
MultipartReader,
ClientSession,
TCPConnector,
)
from aiohttp.abc import AbstractStreamWriter
from aiohttp.hdrs import METH_ANY
from aiohttp.web import Request, StreamResponse, WebSocketResponse
from aiohttp.web_exceptions import HTTPNotFound, HTTPNoContent, HTTPOk, HTTPNotAcceptable, HTTPSeeOther
from aiohttp.web_fileresponse import FileResponse
from aiohttp.web_response import json_response
from aiohttp_swagger3 import SwaggerFile, SwaggerUiSettings
from aiostream import stream
from attrs import evolve
from dateutil import parser as date_parser
from networkx.readwrite import cytoscape_data
from resotocore.analytics import AnalyticsEvent
from resotocore.cli.command import alias_names
from resotocore.cli.model import (
ParsedCommandLine,
CLIContext,
CLICommand,
InternalPart,
WorkerCustomCommand,
AliasTemplate,
InfraAppAlias,
FilePath,
)
from resotocore.config import ConfigValidation, ConfigEntity
from resotocore.console_renderer import ConsoleColorSystem, ConsoleRenderer
from resotocore.db.graphdb import GraphDB, HistoryChange
from resotocore.db.model import QueryModel
from resotocore.dependencies import Dependencies, TenantDependencies
from resotocore.dependencies import TenantDependencyProvider
from resotocore.error import NotFoundError, NotEnoughPermissions
from resotocore.ids import (
TaskId,
ConfigId,
NodeId,
SubscriberId,
WorkerId,
GraphName,
Email,
Password,
valid_root_graph_name,
)
from resotocore.message_bus import Message, ActionDone, Action, ActionError, ActionInfo, ActionProgress
from resotocore.metrics import timed
from resotocore.model.exportable_model import json_export_simple_schema
from resotocore.model.graph_access import Section
from resotocore.model.json_schema import json_schema
from resotocore.model.model import Kind, Model
from resotocore.model.typed_model import to_json, from_js, to_js_str, to_js
from resotocore.query.model import Predicate, PathRoot, variable_to_absolute
from resotocore.query.query_parser import predicate_term
from resotocore.report import Benchmark, ReportCheck
from resotocore.service import Service
from resotocore.task.model import Subscription
from resotocore.types import Json, JsonElement
from resotocore.user.model import Permission, AuthorizedUser
from resotocore.util import uuid_str, force_gen, rnd_str, if_set, duration, utc_str, parse_utc, async_noop, utc
from resotocore.web.auth import raw_jwt_from_auth_message, LoginWithCode, AuthHandler
from resotocore.web.content_renderer import result_binary_gen, single_result
from resotocore.web.directives import (
metrics_handler,
error_handler,
on_response_prepare,
cors_handler,
enable_compression,
default_middleware,
)
from resotocore.web.tsdb import tsdb
from resotocore.worker_task_queue import (
WorkerTaskDescription,
WorkerTask,
WorkerTaskResult,
WorkerTaskInProgress,
)
from resotolib.asynchronous.web.ws_handler import accept_websocket, clean_ws_handler
from resotolib.jwt import encode_jwt
from resotolib.x509 import cert_to_bytes
from resotoui import ui_path
log = logging.getLogger(__name__)
def section_of(request: Request) -> Optional[str]:
section = request.match_info.get("section", request.query.get("section"))
if section and section != "/" and section not in Section.content:
raise AttributeError(f"Given section does not exist: {section}")
return section
# No Authorization required for following paths
AlwaysAllowed = {
"/",
"/.well-known/.*",
"/api-doc.*",
"/authenticate",
"/ca/cert",
"/create-first-user",
"/login",
"/metrics",
"/static/.*",
"/system/.*",
"/ui.*",
"/notebook/.*",
"/debug/.*",
}
# Authorization is not required, but implemented as part of the request handler
DeferredCheck = {"/events", "/work/queue"}
# noinspection PyMethodMayBeStatic
class Api(Service):
def __init__(self, deps: Dependencies, tenant_dependency_provider: TenantDependencyProvider) -> None:
super().__init__()
self.deps = deps
self.tenant_dependency_provider = tenant_dependency_provider
self.auth_handler = AuthHandler(
deps.jwt_signing_key_holder,
deps.config,
deps.cert_handler,
tenant_dependency_provider,
AlwaysAllowed | DeferredCheck,
)
self.app = web.Application(
client_max_size=self.deps.config.api.max_request_size or 1024**2,
# note on order: the middleware is passed in the order provided.
middlewares=[
metrics_handler,
self.auth_handler.middleware(),
cors_handler,
error_handler(deps.config, deps.event_sender),
default_middleware(self),
],
)
self.app.on_response_prepare.append(on_response_prepare)
self._session: Optional[ClientSession] = None
self.in_shutdown = False
self.websocket_handler: Dict[str, Tuple[Future[Any], WebSocketResponse]] = {}
path_part = deps.config.api.web_path.strip().strip("/").strip()
web_path = "" if path_part == "" else f"/{path_part}"
self.__add_routes(web_path)
aiohttp_jinja2.setup(
self.app, loader=jinja2.FileSystemLoader(os.path.abspath(os.path.dirname(__file__) + "/../templates"))
)
@property
def session(self) -> ClientSession:
if self._session is None:
# only keep connections alive for 15 seconds, cleanup closed transports
connector = TCPConnector(keepalive_timeout=15.0, enable_cleanup_closed=True)
self._session = ClientSession(connector=connector)
return self._session
def __add_routes(self, prefix: str) -> None:
static_path = os.path.abspath(os.path.dirname(__file__) + "/../static")
jupyterlite_path = Path(os.path.abspath(os.path.dirname(__file__) + "/../jupyterlite"))
if not jupyterlite_path.exists():
jupyterlite_path.mkdir(parents=True, exist_ok=True)
require = self.auth_handler.allow_with
r = Permission.read
a = Permission.admin
self.app.add_routes(
[
# Model operations (backwards compatible)
web.get(prefix + "/model", require(self.get_model, r)),
web.patch(prefix + "/model", require(self.update_model, a)),
web.get(prefix + "/model/uml", require(self.model_uml, r)),
# Graph based model operations
web.get(prefix + "/graph/{graph_id}/model", require(self.get_model, r)),
web.patch(prefix + "/graph/{graph_id}/model", require(self.update_model, a)),
web.get(prefix + "/graph/{graph_id}/model/uml", require(self.model_uml, r)),
# CRUD Graph operations
web.get(prefix + "/graph", require(self.list_graphs, r)),
web.get(prefix + "/graph/{graph_id}", require(self.get_node, r)),
web.post(prefix + "/graph/{graph_id}", require(self.create_graph, a)),
web.delete(prefix + "/graph/{graph_id}", require(self.wipe, a)),
# search the graph
web.post(prefix + "/graph/{graph_id}/search/raw", require(self.raw, r)),
web.post(prefix + "/graph/{graph_id}/search/structure", require(self.query_structure, r)),
web.post(prefix + "/graph/{graph_id}/search/explain", require(self.explain, r)),
web.post(prefix + "/graph/{graph_id}/search/list", require(self.query_list, r)),
web.post(prefix + "/graph/{graph_id}/search/graph", require(self.query_graph_stream, r)),
web.post(prefix + "/graph/{graph_id}/search/aggregate", require(self.query_aggregation, r)),
web.post(prefix + "/graph/{graph_id}/search/history/list", require(self.query_history, r)),
web.post(prefix + "/graph/{graph_id}/search/history/aggregate", require(self.query_history, r)),
web.post(prefix + "/graph/{graph_id}/property/attributes", require(self.possible_values, r)),
web.post(prefix + "/graph/{graph_id}/property/values", require(self.possible_values, r)),
web.post(prefix + "/graph/{graph_id}/property/path/complete", require(self.property_path_complete, r)),
# maintain the graph
web.patch(prefix + "/graph/{graph_id}/nodes", require(self.update_nodes, a)),
web.post(prefix + "/graph/{graph_id}/merge", require(self.merge_graph, a)),
web.post(prefix + "/graph/{graph_id}/batch/merge", require(self.update_merge_graph_batch, a)),
web.get(prefix + "/graph/{graph_id}/batch", require(self.list_batches, a)),
web.post(prefix + "/graph/{graph_id}/batch/{batch_id}", require(self.commit_batch, a)),
web.delete(prefix + "/graph/{graph_id}/batch/{batch_id}", require(self.abort_batch, a)),
# node specific actions
web.post(
prefix + "/graph/{graph_id}/node/{node_id}/under/{parent_node_id}", require(self.create_node, a)
),
web.get(prefix + "/graph/{graph_id}/node/{node_id}", require(self.get_node, r)),
web.patch(prefix + "/graph/{graph_id}/node/{node_id}", require(self.update_node, a)),
web.delete(prefix + "/graph/{graph_id}/node/{node_id}", require(self.delete_node, a)),
web.patch(prefix + "/graph/{graph_id}/node/{node_id}/section/{section}", require(self.update_node, a)),
# Subscriptions
web.get(prefix + "/subscribers", require(self.list_all_subscriptions, a)),
web.get(prefix + "/subscribers/for/{event_type}", require(self.list_subscription_for_event, a)),
# Subscription
web.get(prefix + "/subscriber/{subscriber_id}", require(self.get_subscriber, a)),
web.put(prefix + "/subscriber/{subscriber_id}", require(self.update_subscriber, a)),
web.delete(prefix + "/subscriber/{subscriber_id}", require(self.delete_subscriber, a)),
web.post(prefix + "/subscriber/{subscriber_id}/{event_type}", require(self.add_subscription, a)),
web.delete(prefix + "/subscriber/{subscriber_id}/{event_type}", require(self.delete_subscription, a)),
web.get(prefix + "/subscriber/{subscriber_id}/handle", require(self.handle_subscribed, a)),
# report checks
web.get(prefix + "/report/checks", require(self.inspection_checks, r)),
web.get(prefix + "/report/benchmarks", require(self.benchmarks, r)),
web.get(prefix + "/report/checks/graph/{graph_id}", require(self.perform_benchmark_on_checks, r)),
web.get(prefix + "/report/check/{check_id}/graph/{graph_id}", require(self.inspection_results, r)),
web.get(prefix + "/report/benchmark/{benchmark}/graph/{graph_id}", require(self.perform_benchmark, r)),
# CLI
web.post(prefix + "/cli/evaluate", require(self.evaluate, r)),
web.post(prefix + "/cli/execute", require(self.execute, r)),
web.get(prefix + "/cli/info", require(self.cli_info, r)),
# Event operations
web.get(prefix + "/events", require(self.handle_events, a)),
web.post(prefix + "/analytics", require(self.send_analytics_events)),
# Worker operations
web.get(prefix + "/work/queue", require(self.handle_work_tasks, a)),
web.get(prefix + "/work/list", require(self.list_work, a)),
# Serve static filed
web.get(prefix, self.forward("/ui/index.html")),
web.static(prefix + "/static", static_path),
web.get(prefix + "/notebook", self.forward("/notebook/index.html")),
web.static(prefix + "/notebook", jupyterlite_path),
# metrics
web.get(prefix + "/metrics", self.metrics),
# config operations
web.get(prefix + "/configs", require(self.list_configs, r)),
web.patch(prefix + "/config/{config_id:[^{}]+}", require(self.patch_config, a)),
web.delete(prefix + "/config/{config_id:[^{}]+}", require(self.delete_config, a)),
# config model operations
web.get(prefix + "/configs/validation", require(self.list_config_models, r)),
web.get(prefix + "/configs/model", require(self.get_configs_model, r)),
web.patch(prefix + "/configs/model", require(self.update_configs_model, a)),
web.put(prefix + "/config_validation/{config_id:[^{}]+}", require(self.put_config_validation, a)),
web.get(prefix + "/config_validation/{config_id:[^{}]+}", require(self.get_config_validation, a)),
web.put(prefix + "/config/{config_id:[^{}]+}/validation", require(self.put_config_validation, a)),
web.get(prefix + "/config/{config_id:[^{}]+}/validation", require(self.get_config_validation, a)),
# config operations, moved here to avoid early matching
web.put(prefix + "/config/{config_id:[^{}]+}", require(self.put_config, a)),
web.get(prefix + "/config/{config_id:[^{}]+}", require(self.get_config, a)),
# ca operations
web.get(prefix + "/ca/cert", self.certificate),
web.post(prefix + "/ca/sign", require(self.sign_certificate, a)),
# system operations
web.get(prefix + "/system/ping", self.ping),
web.get(prefix + "/system/ready", self.ready),
# forwards
web.get(prefix + "/tsdb", self.forward("/tsdb/")),
web.get(prefix + "/ui", self.forward("/ui/index.html")),
web.get(prefix + "/ui/", self.forward("/ui/index.html")),
# auth operations
web.get(prefix + "/.well-known/jwks.json", self.jwks),
web.get(prefix + "/login", require(self.login_page)),
web.post(prefix + "/create-first-user", require(self.create_first_user)),
web.post(prefix + "/authenticate", require(self.authenticate)),
web.get(prefix + "/authorization/user", self.get_authorized_user),
web.get(prefix + "/authorization/renew", self.renew_authorization),
# tsdb operations
web.route(METH_ANY, prefix + "/tsdb/{tail:.+}", tsdb(self)),
web.static(f"{prefix}/ui/", ui_path),
]
)
if self.deps.config.runtime.debug:
self.app.add_routes([web.get(prefix + "/debug/ui/{commit}/{path:.+}", self.serve_debug_ui)])
SwaggerFile(
self.app,
spec_file=f"{static_path}/api-doc.yaml",
swagger_ui_settings=SwaggerUiSettings(path=prefix + "/api-doc", layout="BaseLayout", docExpansion="none"),
)
async def start(self) -> None:
await self.auth_handler.start()
async def stop(self) -> None:
await self.auth_handler.stop()
if not self.in_shutdown:
self.in_shutdown = True
for ws_id in list(self.websocket_handler):
await clean_ws_handler(ws_id, self.websocket_handler)
if self.session:
await self.session.close()
@staticmethod
async def login_with_redirect(request: Request) -> StreamResponse:
params = request.query.copy()
params["redirect"] = request.raw_path
return web.HTTPSeeOther("/login?" + urlencode(params))
@staticmethod
def forward(to: str) -> Callable[[Request], Awaitable[StreamResponse]]:
async def forward_to(request: Request) -> StreamResponse:
goto = to + "?" + urlencode(request.query) if request.query else to
return web.HTTPFound(goto)
return forward_to
@staticmethod
async def ping(_: Request) -> StreamResponse:
return web.HTTPOk(text="pong", content_type="text/plain")
@staticmethod
async def ready(_: Request) -> StreamResponse:
return web.HTTPOk(text="ok")
async def jwks(self, _: Request) -> StreamResponse:
return web.json_response(self.auth_handler.signing_key_jwk)
async def login_page(self, request: Request, deps: TenantDependencies) -> StreamResponse:
template = "login.html" if await deps.user_management.has_users() else "create_first_user.html"
return aiohttp_jinja2.render_template(template, request, context=request.query)
async def create_first_user(self, request: Request, deps: TenantDependencies) -> StreamResponse:
post_data = await request.post()
errors = []
try:
email = Email(str(post_data.get("email", "")).strip())
password = Password(str(post_data.get("password", "")))
password_repeat = str(post_data.get("password_repeat", ""))
company = str(post_data.get("company", "")).strip()
fullname = str(post_data.get("fullname", "")).strip()
if not email or email.startswith("@") or email.endswith("@") or email.count("@") != 1:
errors.append("Invalid email address")
if not password:
errors.append("Password is required")
if not company:
errors.append("Company name is required")
if not fullname:
errors.append("Full name is required")
if password != password_repeat:
errors.append("Passwords do not match")
if not errors:
await deps.user_management.create_first_user(company, fullname, email, password)
return await self.authenticate(request, deps)
except Exception as e:
errors.append(str(e))
error_string = ". ".join(errors)
return aiohttp_jinja2.render_template(
"create_first_user.html", request, context={**post_data, "error": error_string}
)
async def authenticate(self, request: Request, deps: TenantDependencies) -> StreamResponse:
post_data = await request.post()
email = Email(str(post_data.get("email", "")))
password = Password(str(post_data.get("password", "")))
redirect = str(post_data.get("redirect", ""))
if email and password and (user := await deps.user_management.login(email, password)):
params: Dict[str, List[str]] = {}
if self.deps.config.args.psk:
code = await self.auth_handler.add_login_with_code(LoginWithCode(email, user.roles, utc()))
params["code"] = [code]
if redirect:
if params:
parsed = urlparse(redirect)
query_params = parse_qs(parsed.query)
query_params.update(params)
parsed = parsed._replace(query=urlencode(query_params, doseq=True))
redirect = urlunparse(parsed)
response: StreamResponse = HTTPSeeOther(redirect)
else:
response = HTTPOk(text=urlencode(params, doseq=True))
return response
return aiohttp_jinja2.render_template(
"login.html", request, context=dict(**post_data, error="Invalid username or password")
)
@staticmethod
async def get_authorized_user(request: Request) -> StreamResponse:
if jwt := request.get("jwt"):
return web.json_response(jwt)
else:
return web.HTTPNoContent()
async def renew_authorization(self, request: Request) -> StreamResponse:
if jwt_raw := request.get("jwt"):
exp = datetime.fromtimestamp(int(jwt_raw["exp"]), tz=timezone.utc)
user = LoginWithCode(jwt_raw["email"], set(jwt_raw["roles"].split(",")), exp)
renewed, data = self.auth_handler.user_jwt(user)
return web.json_response(data, headers={"Authorization": f"Bearer {renewed}"})
else:
return HTTPNoContent() # no psk, no renewal
async def list_configs(self, request: Request, deps: TenantDependencies) -> StreamResponse:
return await self.stream_response_from_gen(request, deps.config_handler.list_config_ids())
async def get_config(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = ConfigId(request.match_info["config_id"])
accept = request.headers.get("accept", "application/json")
not_found = HTTPNotFound(text="No config with this id")
if accept == "application/yaml":
yml = await deps.config_handler.config_yaml(config_id)
return web.Response(body=yml.encode("utf-8"), content_type="application/yaml") if yml else not_found
else:
def get_query_param_bool(name: str, default: bool) -> bool:
return request.query.get(name, "true" if default else "false").lower() == "true"
# do we want the config with overrides/env_vars applied in-place or in a separate object?
separate_overrides = get_query_param_bool("separate_overrides", default=False)
if separate_overrides:
# if we want separate overrides, we don't apply overrides to the existing config
# and don't substitute the env vars by default. E.g. UI asks us for the config
apply_overrides = get_query_param_bool("apply_overrides", default=False)
resolve_env_vars = get_query_param_bool("resolve_env_vars", default=False)
# attach the "raw" config version that was stored in the database
include_raw_config = get_query_param_bool("include_raw_config", default=False)
else:
# if we request a single object with overrides applied,
# we apply overrides and resolve env vars by default
apply_overrides = get_query_param_bool("apply_overrides", default=True)
resolve_env_vars = get_query_param_bool("resolve_env_vars", default=True)
# ignored in case of a single config object requested
include_raw_config = False
config = await deps.config_handler.get_config(config_id, apply_overrides, resolve_env_vars)
if config:
headers = {"Resoto-Config-Revision": config.revision}
if separate_overrides:
payload = {"config": config.config, "overrides": deps.config_override.get_override(config_id)}
if include_raw_config:
raw_config = await deps.config_handler.get_config(
config_id, apply_overrides=False, resolve_env_vars=False
)
payload["raw_config"] = raw_config.config if raw_config else None
else:
payload = config.config
return await single_result(request, payload, headers)
else:
return not_found
async def put_config(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = ConfigId(request.match_info["config_id"])
validate = request.query.get("validate", "true").lower() != "false"
dry_run = request.query.get("dry_run", "false").lower() == "true"
config = await self.json_from_request(request)
result = await deps.config_handler.put_config(
ConfigEntity(config_id, config), validate=validate, dry_run=dry_run
)
headers = {"Resoto-Config-Revision": result.revision}
return await single_result(request, result.config, headers)
async def patch_config(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = ConfigId(request.match_info["config_id"])
validate = request.query.get("validate", "true").lower() != "false"
dry_run = request.query.get("dry_run", "false").lower() == "true"
patch = await self.json_from_request(request)
updated = await deps.config_handler.patch_config(
ConfigEntity(config_id, patch), validate=validate, dry_run=dry_run
)
headers = {"Resoto-Config-Revision": updated.revision}
return await single_result(request, updated.config, headers)
async def delete_config(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = ConfigId(request.match_info["config_id"])
await deps.config_handler.delete_config(config_id)
return HTTPNoContent()
async def list_config_models(self, request: Request, deps: TenantDependencies) -> StreamResponse:
return await self.stream_response_from_gen(request, deps.config_handler.list_config_validation_ids())
async def get_config_validation(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = request.match_info["config_id"]
model = await deps.config_handler.get_config_validation(config_id)
return await single_result(request, to_js(model)) if model else HTTPNotFound(text="No model for this config.")
async def get_configs_model(self, request: Request, deps: TenantDependencies) -> StreamResponse:
model = await deps.config_handler.get_configs_model()
if request.query.get("flat", "false") == "true":
model = model.flat_kinds()
return await single_result(request, to_js(model, strip_nulls=True))
async def update_configs_model(self, request: Request, deps: TenantDependencies) -> StreamResponse:
js = await self.json_from_request(request)
kinds: List[Kind] = from_js(js, List[Kind])
model = await deps.config_handler.update_configs_model(kinds)
return await single_result(request, to_js(model))
async def put_config_validation(self, request: Request, deps: TenantDependencies) -> StreamResponse:
config_id = request.match_info["config_id"]
js = await self.json_from_request(request)
js["id"] = config_id
config_model = from_js(js, ConfigValidation)
model = await deps.config_handler.put_config_validation(config_model)
return await single_result(request, to_js(model))
async def certificate(self, _: Request) -> StreamResponse:
cert, fingerprint = self.deps.cert_handler.authority_certificate
headers = {
"SHA256-Fingerprint": fingerprint,
"Content-Disposition": 'attachment; filename="resoto_root_ca.pem"',
}
if self.deps.config.args.psk:
headers["Authorization"] = "Bearer " + encode_jwt(
{"sha256_fingerprint": fingerprint}, self.deps.config.args.psk
)
return HTTPOk(headers=headers, body=cert, content_type="application/x-pem-file")
async def sign_certificate(self, request: Request, _: TenantDependencies) -> StreamResponse:
csr_bytes = await request.content.read()
cert, fingerprint = self.deps.cert_handler.sign(csr_bytes)
headers = {"SHA256-Fingerprint": fingerprint}
return HTTPOk(headers=headers, body=cert_to_bytes(cert), content_type="application/x-pem-file")
@staticmethod
async def metrics(_: Request) -> StreamResponse:
resp = web.Response(body=prometheus_client.generate_latest())
resp.content_type = prometheus_client.CONTENT_TYPE_LATEST
return resp
async def list_all_subscriptions(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscribers = await deps.subscription_handler.all_subscribers()
return await single_result(request, to_json(subscribers))
async def get_subscriber(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
subscriber = await deps.subscription_handler.get_subscriber(subscriber_id)
return self.optional_json(subscriber, f"No subscriber with id {subscriber_id}")
async def list_subscription_for_event(self, request: Request, deps: TenantDependencies) -> StreamResponse:
event_type = request.match_info["event_type"]
subscribers = await deps.subscription_handler.list_subscriber_for(event_type)
return await single_result(request, to_json(subscribers))
async def update_subscriber(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
body = await self.json_from_request(request)
subscriptions = from_js(body, List[Subscription])
sub = await deps.subscription_handler.update_subscriptions(subscriber_id, subscriptions)
return await single_result(request, to_json(sub))
async def delete_subscriber(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
await deps.subscription_handler.remove_subscriber(subscriber_id)
return web.HTTPNoContent()
async def add_subscription(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
event_type = request.match_info["event_type"]
timeout = timedelta(seconds=int(request.query.get("timeout", "60")))
wait_for_completion = request.query.get("wait_for_completion", "true").lower() != "false"
sub = await deps.subscription_handler.add_subscription(subscriber_id, event_type, wait_for_completion, timeout)
return await single_result(request, to_js(sub))
async def delete_subscription(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
event_type = request.match_info["event_type"]
sub = await deps.subscription_handler.remove_subscription(subscriber_id, event_type)
return await single_result(request, to_js(sub))
async def handle_subscribed(self, request: Request, deps: TenantDependencies) -> StreamResponse:
subscriber_id = SubscriberId(request.match_info["subscriber_id"])
subscriber = await deps.subscription_handler.get_subscriber(subscriber_id)
if subscriber_id in self.deps.message_bus.active_listener:
log.info(f"There is already a listener for subscriber: {subscriber_id}. Reject.")
return web.HTTPTooManyRequests(text="Only one connection per subscriber is allowed!")
elif subscriber and subscriber.subscriptions:
pending = await deps.task_handler.list_all_pending_actions_for(subscriber)
return await self.listen_to_events(
request, deps, subscriber_id, list(subscriber.subscriptions.keys()), pending
)
else:
return web.HTTPNotFound(text=f"No subscriber with this id: {subscriber_id} or no subscriptions")
async def perform_benchmark_on_checks(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph = GraphName(request.match_info["graph_id"])
provider = request.query.get("provider")
service = request.query.get("service")
category = request.query.get("category")
kind = request.query.get("kind")
acc = request.query.get("accounts")
accounts = [a.strip() for a in acc.split(",")] if acc else None
result = await deps.inspector.perform_checks(
graph, provider=provider, service=service, category=category, kind=kind, accounts=accounts
)
return await single_result(request, to_js(result))
async def perform_benchmark(self, request: Request, deps: TenantDependencies) -> StreamResponse:
benchmark = request.match_info["benchmark"]
graph = GraphName(request.match_info["graph_id"])
acc = request.query.get("accounts")
accounts = [a.strip() for a in acc.split(",")] if acc else None
action = request.query.get("action", "run")
if action == "run":
results = await deps.inspector.perform_benchmarks(graph, [benchmark], accounts=accounts)
elif action == "load":
results = await deps.inspector.load_benchmarks(graph, [benchmark], accounts=accounts)
else:
raise ValueError(f"Unknown action {action}. One of run or load is expected.")
result_graph = results[benchmark].to_graph()
async with stream.iterate(result_graph).stream() as streamer:
return await self.stream_response_from_gen(request, streamer, len(result_graph))
async def inspection_checks(self, request: Request, deps: TenantDependencies) -> StreamResponse:
provider = request.query.get("provider")
service = request.query.get("service")
category = request.query.get("category")
kind = request.query.get("kind")
check_ids = request.query["id"].split(",") if "id" in request.query else None
inspections = await deps.inspector.list_checks(
provider=provider, service=service, category=category, kind=kind, check_ids=check_ids
)
return await single_result(request, to_js(inspections))
async def benchmarks(self, request: Request, deps: TenantDependencies) -> StreamResponse:
benchmark_filter = [b.strip() for b in request.query.get("benchmarks", "").split(",") if b.strip()]
short = request.query.get("short", "false").lower() == "true"
with_checks = request.query.get("with_checks", "false").lower() == "true"
lookup = {c.id: c for c in await deps.inspector.list_checks()} if with_checks else {}
def to_js_check(c: ReportCheck) -> JsonElement:
return dict(id=c.id, severity=c.severity.value) if short else to_js(c, strip_nulls=True)
def to_js_benchmark(b: Benchmark) -> Json:
bj: Json = to_js(b, strip_nulls=True)
if short:
bj.pop("checks", None)
bj.pop("children", None)
if with_checks:
bj["report_checks"] = [to_js_check(lookup[c]) for c in b.nested_checks()]
return bj
benchmarks = [
to_js_benchmark(b)
for b in await deps.inspector.list_benchmarks()
if (b.id in benchmark_filter or not benchmark_filter)
]
return await single_result(request, benchmarks)
async def inspection_results(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph = GraphName(request.match_info["graph_id"])
check_id = request.match_info["check_id"]
acc = request.query.get("accounts")
accounts = [a.strip() for a in acc.split(",")] if acc else None
inspections = await deps.inspector.list_failing_resources(graph, check_id, accounts)
return await self.stream_response_from_gen(request, inspections)
async def handle_events(self, request: Request, deps: TenantDependencies) -> StreamResponse:
show = request.query["show"].split(",") if "show" in request.query else ["*"]
return await self.listen_to_events(request, deps, SubscriberId(str(uuid.uuid1())), show)
async def send_analytics_events(self, request: Request, _: TenantDependencies) -> StreamResponse:
events_json = await self.json_from_request(request)
events = from_js(events_json, List[AnalyticsEvent])
await self.deps.event_sender.capture(events)
return web.HTTPNoContent()
async def listen_to_events(
self,
request: Request,
deps: TenantDependencies,
listener_id: SubscriberId,
event_types: List[str],
initial_messages: Optional[Sequence[Message]] = None,
) -> WebSocketResponse:
handler: Callable[[str], Awaitable[None]] = async_noop
async def authorize_request(msg: str) -> None:
nonlocal handler
if (r := raw_jwt_from_auth_message(msg)) and await self.auth_handler.validate_jwt(r, request) is not None:
handler = handle_message
else:
raise ValueError("No Authorization header provided and no valid auth message sent")
async def handle_message(msg: str) -> None:
js = json.loads(msg)
if "data" in js:
js["data"]["subscriber_id"] = listener_id
js["data"]["received_at"] = utc_str()
message: Message = from_js(js, Message)
if isinstance(message, Action):
raise AttributeError("Actors should not emit action messages. ")
elif isinstance(message, ActionInfo):
await deps.task_handler.handle_action_info(message)
elif isinstance(message, ActionProgress):
await deps.task_handler.handle_action_progress(message)
elif isinstance(message, ActionDone):
await deps.task_handler.handle_action_done(message)
elif isinstance(message, ActionError):
await deps.task_handler.handle_action_error(message)
else:
await deps.message_bus.emit(message)
handler = authorize_request if request.get("authorized", False) is False else handle_message
return await accept_websocket(
request,
handle_incoming=lambda x: handler(x), # pylint: disable=unnecessary-lambda # it is required!
outgoing_context=partial(self.deps.message_bus.subscribe, listener_id, event_types),
websocket_handler=self.websocket_handler,
initial_messages=initial_messages,
)
async def handle_work_tasks(self, request: Request, deps: TenantDependencies) -> WebSocketResponse:
worker_id = WorkerId(uuid_str())
worker_descriptions: Future[List[WorkerTaskDescription]] = asyncio.get_event_loop().create_future()
handler: Callable[[str], Awaitable[None]] = async_noop
async def authorize_request(msg: str) -> None:
nonlocal handler
if (r := raw_jwt_from_auth_message(msg)) and await self.auth_handler.validate_jwt(r, request) is not None:
handler = handle_connect
else:
raise ValueError("No Authorization header provided and no valid auth message sent")
async def handle_connect(msg: str) -> None:
nonlocal handler
cmds = from_js(json.loads(msg), List[WorkerCustomCommand])
description = [WorkerTaskDescription(cmd.name, cmd.filter) for cmd in cmds]
# set the future and allow attaching the worker to the task queue
worker_descriptions.set_result(description)
# register the descriptions as custom command on the CLI
for cmd in cmds:
deps.cli.register_alias_template(cmd.to_template())
# the connect process is done, define the final handler
handler = handle_message
async def handle_message(msg: str) -> None:
tr = from_js(json.loads(msg), WorkerTaskResult)
if tr.result == "error":
error = tr.error if tr.error else "worker signalled error without detailed error message"
await self.deps.worker_task_queue.error_task(worker_id, tr.task_id, error)
elif tr.result == "done":
await self.deps.worker_task_queue.acknowledge_task(worker_id, tr.task_id, tr.data)
else:
log.info(f"Do not understand this message: {msg}")
def task_json(task: WorkerTask) -> str:
return to_js_str(task.to_json())
@asynccontextmanager
async def connect_to_task_queue() -> AsyncIterator[Queue[WorkerTask]]:
# we need to wait for the worker to send the list of commands it can handle
# before we can attach to the worker task queue
descriptions = await worker_descriptions
async with self.deps.worker_task_queue.attach(worker_id, descriptions) as queue:
yield queue
handler = authorize_request if request.get("authorized", False) is False else handle_connect
# noinspection PyTypeChecker
return await accept_websocket(
request,
handle_incoming=lambda x: handler(x), # pylint: disable=unnecessary-lambda # it is required!
outgoing_context=connect_to_task_queue,
websocket_handler=self.websocket_handler,
outgoing_fn=task_json,
)
async def list_work(self, _: Request, __: TenantDependencies) -> StreamResponse:
def wt_to_js(ip: WorkerTaskInProgress) -> Json:
return {
"task": ip.task.to_json(),
"worker": ip.worker.worker_id,
"retry_counter": ip.retry_counter,
"deadline": to_json(ip.deadline),
}
return web.json_response([wt_to_js(ot) for ot in self.deps.worker_task_queue.outstanding_tasks.values()])
async def model_uml(self, request: Request, deps: TenantDependencies) -> StreamResponse:
output = request.query.get("output", "svg")
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
show = request.query["show"].split(",") if "show" in request.query else None
hide = request.query["hide"].split(",") if "hide" in request.query else None
with_inheritance = request.query.get("with_inheritance", "true") != "false"
with_base_classes = request.query.get("with_base_classes", "true") != "false"
with_subclasses = request.query.get("with_subclasses", "false") != "false"
dependency = set(request.query["dependency"].split(",")) if "dependency" in request.query else None
with_predecessors = request.query.get("with_predecessors", "false") != "false"
with_successors = request.query.get("with_successors", "false") != "false"
with_properties = request.query.get("with_properties", "true") != "false"
aggregate_roots = request.query.get("aggregate_roots", "true") != "false"
link_classes = request.query.get("link_classes", "false") != "false"
sort_props = request.query.get("sort_props", "true") != "false"
result = await deps.model_handler.uml_image(
graph_name=graph_id,
output=output,
show_packages=show,
hide_packages=hide,
with_inheritance=with_inheritance,
with_base_classes=with_base_classes,
with_subclasses=with_subclasses,
dependency_edges=dependency, # type: ignore
with_predecessors=with_predecessors,
with_successors=with_successors,
with_properties=with_properties,
link_classes=link_classes,
only_aggregate_roots=aggregate_roots,
sort_props=sort_props,
)
response = web.StreamResponse()
mt = {"svg": "image/svg+xml", "png": "image/png", "puml": "text/plain"}
response.headers["Content-Type"] = mt[output]
await response.prepare(request)
await response.write_eof(result)
return response
async def get_model(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
full_model = await deps.model_handler.load_model(graph_id)
with_bases = if_set(request.query.get("with_bases"), lambda x: x.lower() == "true", False)
with_property_kinds = if_set(request.query.get("with_property_kinds"), lambda x: x.lower() == "true", False)
with_properties = if_set(request.query.get("with_properties"), lambda x: x.lower() == "true", True)
with_relatives = if_set(request.query.get("with_relatives"), lambda x: x.lower() == "true", True)
with_metadata = if_set(request.query.get("with_metadata"), lambda x: x.lower() == "true", True)
aggregate_roots_only = if_set(request.query.get("aggregate_roots_only"), lambda x: x.lower() == "true", False)
md = full_model
if kind := request.query.get("kind"):
kinds = set(kind.split(","))
md = md.filter_complex(lambda x: x.fqn in kinds, with_bases, with_property_kinds)
if filter_names := request.query.get("filter"):
parts = filter_names.split(",")
md = md.filter_complex(lambda x: any(x.fqn in p for p in parts), with_bases, with_property_kinds)
if aggregate_roots_only:
md = md.filter_complex(lambda x: x.aggregate_root, with_bases, with_property_kinds)
md = md.flat_kinds(full_model) if request.query.get("flat", "false") == "true" else md
export_format = request.query.get("format")
# default to internal model format, but allow requesting json schema format
if export_format == "schema" or request.headers.get("accept") == "application/schema+json":
return json_response(json_schema(md), content_type="application/schema+json")
elif export_format == "simple":
return await single_result(
request, json_export_simple_schema(md, with_properties, with_relatives, with_metadata)
)
else:
json_model = [
m.as_json(with_properties=with_properties, with_relatives=with_relatives, with_metadata=with_metadata)
for m in md.kinds.values()
]
return await single_result(request, json.loads(json.dumps(json_model, sort_keys=True)))
async def update_model(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
js = await self.json_from_request(request)
kinds: List[Kind] = from_js(js, List[Kind])
model = await deps.model_handler.update_model(graph_id, kinds)
return await single_result(request, to_js(model, strip_nulls=True))
async def get_node(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
node_id = NodeId(request.match_info.get("node_id", "root"))
graph = deps.db_access.get_graph_db(graph_id)
model = await deps.model_handler.load_model(graph_id)
node = await graph.get_node(model, node_id)
if node is None:
return web.HTTPNotFound(text=f"No such node with id {node_id} in graph {graph_id}")
else:
return await single_result(request, node)
async def create_node(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
node_id = NodeId(request.match_info.get("node_id", "some_existing"))
parent_node_id = NodeId(request.match_info.get("parent_node_id", "root"))
graph = deps.db_access.get_graph_db(graph_id)
item = await self.json_from_request(request)
md = await deps.model_handler.load_model(graph_id)
node = await graph.create_node(md, node_id, item, parent_node_id)
return await single_result(request, node)
async def update_node(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
node_id = NodeId(request.match_info.get("node_id", "some_existing"))
section = section_of(request)
graph = deps.db_access.get_graph_db(graph_id)
patch = await self.json_from_request(request)
md = await deps.model_handler.load_model(graph_id)
node = await graph.update_node(md, node_id, patch, False, section)
return await single_result(request, node)
async def delete_node(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_name = GraphName(request.match_info.get("graph_id", "resoto"))
node_id = NodeId(request.match_info.get("node_id", "some_existing"))
if node_id == "root":
raise AttributeError("Root node can not be deleted!")
graph = deps.db_access.get_graph_db(graph_name)
model = await deps.model_handler.load_model(graph_name)
await graph.delete_node(node_id, model)
return web.HTTPNoContent()
async def update_nodes(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_name = GraphName(request.match_info.get("graph_id", "resoto"))
allowed = {*Section.content, "id", "revision"}
updates: Dict[NodeId, Json] = {}
async for elem in self.to_json_generator(request):
keys = set(elem.keys())
assert keys.issubset(allowed), f"Invalid json. Allowed keys are: {allowed}"
assert "id" in elem, f"No id given for element {elem}"
assert keys.intersection(Section.content), f"No update provided for element {elem}"
uid = elem["id"]
assert uid not in updates, f"Only one update allowed per id! {elem}"
del elem["id"]
updates[uid] = elem
db = deps.db_access.get_graph_db(graph_name)
model = await deps.model_handler.load_model(graph_name)
result_gen = db.update_nodes(model, updates)
return await self.stream_response_from_gen(request, result_gen)
async def list_graphs(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graphs = await deps.db_access.list_graphs()
return await single_result(request, graphs)
async def create_graph(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_name = GraphName(request.match_info.get("graph_id", "resoto"))
if valid_root_graph_name(graph_name) is False:
raise AttributeError("Graph name is not valid (no underscores, can not start with snapshot-)")
graph = await deps.db_access.create_graph(graph_name)
model = await deps.model_handler.load_model(graph_name)
root = await graph.get_node(model, NodeId("root"))
return web.json_response(root)
async def merge_graph(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
wait_for_result = request.query.get("wait_for_result", "true").lower() == "true"
task_id: Optional[TaskId] = None
if tid := request.headers.get("Resoto-Worker-Task-Id"):
task_id = TaskId(tid)
log.info(
f"Received merge_graph request for graph {graph_id}, wait_for_result={wait_for_result}, task_id={task_id}"
)
db = deps.db_access.get_graph_db(graph_id)
it = self.to_line_generator(request)
max_wait = self.deps.config.graph_update.merge_max_wait_time()
info = await deps.graph_merger.merge_graph(db, it, max_wait, None, task_id, wait_for_result)
return web.json_response(to_js(info)) if info else web.HTTPNoContent()
async def update_merge_graph_batch(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_id = GraphName(request.match_info.get("graph_id", "resoto"))
wait_for_result = request.query.get("wait_for_result", "true").lower() == "true"
task_id: Optional[TaskId] = None
if tid := request.headers.get("Resoto-Worker-Task-Id"):
task_id = TaskId(tid)
log.info(f"Received put_sub_graph_batch request for graph {graph_id}, wait_for_result={wait_for_result}")
db = deps.db_access.get_graph_db(graph_id)
rnd = "".join(SystemRandom().choice(string.ascii_letters) for _ in range(12))
batch_id = request.query.get("batch_id", rnd)
it = self.to_line_generator(request)
max_wait = self.deps.config.graph_update.merge_max_wait_time()
info = await deps.graph_merger.merge_graph(db, it, max_wait, batch_id, task_id, wait_for_result)
headers = {"BatchId": batch_id}
return web.json_response(to_json(info), headers=headers) if info else web.HTTPNoContent(headers=headers)
async def list_batches(self, request: Request, deps: TenantDependencies) -> StreamResponse:
graph_db = deps.db_access.get_graph_db(GraphName(request.match_info.get("graph_id", "resoto")))