From 73d816d21167d2d03415f14af961eab35de4b661 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Wed, 16 Nov 2022 14:55:16 +0200 Subject: [PATCH 01/55] initial commit - validation models defined --- .../curieconf/confserver/v3/fast_api.py | 752 ++++++++++++++++++ 1 file changed, 752 insertions(+) create mode 100644 curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py new file mode 100644 index 000000000..0dcff4755 --- /dev/null +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -0,0 +1,752 @@ +import datetime +import typing +from typing import Optional, List, Union + +from fastapi import FastAPI, Request +from pydantic import BaseModel, Field, validator +import random # needed for generating a random number for an API +import uvicorn # optional if you run it directly from terminal +import jsonschema + +# monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type +jsonschema.Draft4Validator = jsonschema.Draft3Validator + +from curieconf import utils +from curieconf.utils import cloud +from curieconf.confserver import app + +import requests +from jsonschema import validate +from pathlib import Path +import json + +# api_bp = Blueprint("api_v3", __name__) +# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") + +# ns_configs = api.namespace("configs", description="Configurations") +# ns_db = api.namespace("db", description="Database") +# ns_tools = api.namespace("tools", description="Tools") + + +############## +### MODELS ### +############## + + +### Models for documents +anyTypeUnion = Union[int, float, bool, object, list, None] +anyOp = Optional[object] +anyType = ["number", "string", "boolean", "object", "array", "null"] + + +# class AnyType(fields.Raw): +# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] + + +# limit + +class Threshold(BaseModel): + limit: int + action: str + + +# m_threshold = api.model( +# "Rate Limit Threshold", +# { +# "limit": fields.Integer(required=True), +# "action": fields.String(required=True), +# }, +# ) + +class Limit(BaseModel): + id: str + name: str + description: Optional[str] + _global: bool = Field(alias="global") + active: bool + timeframe: int + thresholds: List[Threshold] + include: typing.Any + exclude: typing.Any + key: anyTypeUnion + pairwith: typing.Any + tags: List[str] + + +# m_limit = api.model( +# "Rate Limit", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "global": fields.Boolean(required=True), +# "active": fields.Boolean(required=True), +# "timeframe": fields.Integer(required=True), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "include": fields.Raw(required=True), +# "exclude": fields.Raw(required=True), +# "key": AnyType(required=True), +# "pairwith": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# }, +# ) + +# securitypolicy +class SecProfileMap(BaseModel): + id: str + name: str + description: str + match: str + acl_profile: str + acl_active: bool + content_filter_profile: str + content_filter_active: bool + limit_ids: Optional[list] + + +# m_secprofilemap = api.model( +# "Security Profile Map", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.String(required=True), +# "acl_profile": fields.String(required=True), +# "acl_active": fields.Boolean(required=True), +# "content_filter_profile": fields.String(required=True), +# "content_filter_active": fields.Boolean(required=True), +# "limit_ids": fields.List(fields.Raw()), +# }, +# ) + +# TODO = deprecated? +# m_map = api.model( +# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} +# ) + +class SecurityPolicy(BaseModel): + id: str + name: str + description: str + tags: List[str] + match: str + session: anyTypeUnion + session_ids: anyTypeUnion + map: List[SecProfileMap] + + +# m_securitypolicy = api.model( +# "Security Policy", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String()), +# "match": fields.String(required=True), +# "session": AnyType(), +# "session_ids": AnyType(), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# }, +# ) + +# content filter rule + +class ContentFilterRule(BaseModel): + id: str + name: str + msg: str + operand: str + severity: int + certainity: int + category: str + subcategory: str + risk: int + tags: Optional[List[str]] + description: Optional[str] + + +# m_contentfilterrule = api.model( +# "Content Filter Rule", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "msg": fields.String(required=True), +# "operand": fields.String(required=True), +# "severity": fields.Integer(required=True), +# "certainity": fields.Integer(required=True), +# "category": fields.String(required=True), +# "subcategory": fields.String(required=True), +# "risk": fields.Integer(required=True), +# "tags": fields.List(fields.String()), +# "description": fields.String(), +# }, +# ) + +# content filter profile +class ContentFilterProfile(BaseModel): + id: str + name: str + description: Optional[str] + ignore_alphanum: bool + args: typing.Any + headers: typing.Any + cookies: typing.Any + path: typing.Any + allsections: typing.Any + decoding: typing.Any + masking_seed: str + content_type: Optional[List[str]] + active: Optional[List[str]] + report: Optional[List[str]] + ignore: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + ignore_body: bool + + +# m_contentfilterprofile = api.model( +# "Content Filter Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "ignore_alphanum": fields.Boolean(required=True), +# "args": fields.Raw(required=True), +# "headers": fields.Raw(required=True), +# "cookies": fields.Raw(required=True), +# "path": fields.Raw(required=True), +# "allsections": fields.Raw(), +# "decoding": fields.Raw(required=True), +# "masking_seed": fields.String(required=True), +# "content_type": fields.List(fields.String()), +# "active": fields.List(fields.String()), +# "report": fields.List(fields.String()), +# "ignore": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# "ignore_body": fields.Boolean(required=True), +# }, +# ) + +# aclprofile +class ACLProfile(BaseModel): + id: str + name: str + description: Optional[str] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + + +# m_aclprofile = api.model( +# "ACL Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# }, +# ) + +# Global Filter +class GlobalFilter(BaseModel): + id: str + name: str + source: str + mdate: str + description: str + active: bool + action: typing.Any + tags: Optional[List[str]] + rule: anyTypeUnion + + +# m_glbalfilter = api.model( +# "Global Filter", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "source": fields.String(required=True), +# "mdate": fields.String(required=True), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# "action": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# "rule": AnyType(), +# }, +# ) + +# Flow Control + +class FlowControl(BaseModel): + id: str + name: str + timeframe: int + key: List[typing.Any] + sequence: List[typing.Any] + tags: Optional[List[str]] + include: Optional[List[str]] + exclude: Optional[List[str]] + description: Optional[str] + active: bool + + +# +# m_flowcontrol = api.model( +# "Flow Control", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "timeframe": fields.Integer(required=True), +# "key": fields.List(fields.Raw(required=True)), +# "sequence": fields.List(fields.Raw(required=True)), +# "tags": fields.List(fields.String()), +# "include": fields.List(fields.String()), +# "exclude": fields.List(fields.String()), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# }, +# ) + +# Action + +class Action(BaseModel): + id: str + name: str + description: Optional[str] + tags: List[str] + params: typing.Any + type: str + + +# m_action = api.model( +# "Action", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String(required=True)), +# "params": fields.Raw(), +# "type": fields.String(required=True), +# }, +# ) + +# Virtual Tag +class VirtualTag(BaseModel): + id: str + name: str + description: Optional[str] + match: List[typing.Any] + + +# +# m_virtualtag = api.model( +# "Virtual Tag", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.List(fields.Raw(required=True)), +# }, +# ) + +# custom +class Custom(BaseModel): + id: str + name: str + + +# m_custom = api.model( +# "Custom", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +### mapping from doc name to model + +models = { + "ratelimits": Limit, + "securitypolicies": SecurityPolicy, + "contentfilterrules": ContentFilterRule, + "contentfilterprofiles": ContentFilterProfile, + "aclprofiles": ACLProfile, + "globalfilters": GlobalFilter, + "flowcontrol": FlowControl, + "actions": Action, + "virtualtags": Custom, + "custom": Custom, +} + + +### Other models +class DocumentMask(BaseModel): + id: str + name: str + description: str + map: Optional[List[SecProfileMap]] + include: Optional[List[typing.Any]] + exclude: Optional[List[typing.Any]] + tags: Optional[List[str]] + active: Optional[List[typing.Any]] + action: typing.Any + sequence: Optional[List[typing.Any]] + timeframe: Optional[int] + thresholds: Optional[List[Threshold]] + pairwith: typing.Any + content_type: Optional[List[str]] + params: typing.Any + decoding: typing.Any + category: Optional[str] + subcategory: Optional[str] + risk: Optional[int] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + match: Optional[str] = "j" + _type: Optional[str] = Field(alias="type") + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# m_document_mask = api.model( +# "Mask for document", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(required=True), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# "include": fields.Wildcard(fields.Raw()), +# "exclude": fields.Wildcard(fields.Raw()), +# "tags": fields.List(fields.String()), +# "active": fields.Wildcard(fields.Raw()), +# "action": fields.Raw(), +# "sequence": fields.List(fields.Raw()), +# "timeframe": fields.Integer(), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "pairwith": fields.Raw(), +# "content_type": fields.List(fields.String()), +# "params": fields.Raw(), +# "decoding": fields.Raw(), +# "category": fields.String(), +# "subcategory": fields.String(), +# "risk": fields.Integer(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "match": fields.String(), +# "type": fields.String(), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class VersionLog(BaseModel): + version: Optional[str] + # TODO - dt_format="iso8601" + date: Optional[datetime.datetime] + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# +# m_version_log = api.model( +# "Version log", +# { +# "version": fields.String(), +# "date": fields.DateTime(dt_format="iso8601"), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class Meta(BaseModel): + id: str + description: str + date: Optional[datetime.datetime] + logs: Optional[List[VersionLog]] = [] + version: Optional[str] + + +# m_meta = api.model( +# "Meta", +# { +# "id": fields.String(required=True), +# "description": fields.String(required=True), +# "date": fields.DateTime(), +# "logs": fields.List(fields.Nested(m_version_log), default=[]), +# "version": fields.String(), +# }, +# ) + +class BlobEntry(BaseModel): + format: str + blob: anyTypeUnion + + +# m_blob_entry = api.model( +# "Blob Entry", +# { +# "format": fields.String(required=True), +# "blob": AnyType(), +# }, +# ) + +class BlobListEntry(BaseModel): + name: Optional[str] + + +# m_blob_list_entry = api.model( +# "Blob ListEntry", +# { +# "name": fields.String(), +# }, +# ) + +class DocumentListEntry(BaseModel): + name: Optional[str] + entries: Optional[int] + + +# m_document_list_entry = api.model( +# "Document ListEntry", +# { +# "name": fields.String(), +# "entries": fields.Integer(), +# }, +# ) + +class ConfigDocuments(BaseModel): + ratelimits: Optional[List[models["ratelimits"]]] = [] + securitypolicies: Optional[List[models["securitypolicies"]]] = [] + contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] + contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] + aclprofiles: Optional[List[models["aclprofiles"]]] = [] + globalfilters: Optional[List[models["globalfilters"]]] = [] + flowcontrol: Optional[List[models["flowcontrol"]]] = [] + actions: Optional[List[models["actions"]]] = [] + virtualtags: Optional[List[models["virtualtags"]]] = [] + custom: Optional[List[models["custom"]]] = [] + + +# m_config_documents = api.model( +# "Config Documents", +# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, +# ) + + +class ConfigBlobs(BaseModel): + geolite2asn: Optional[List[Optional[BlobEntry]]] + geolite2country: Optional[List[Optional[BlobEntry]]] + geolite2city: Optional[List[Optional[BlobEntry]]] + customconf: Optional[List[Optional[BlobEntry]]] + + +# m_config_blobs = api.model( +# "Config Blobs", +# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, +# ) + +class ConfigDeleteBlobs(BaseModel): + geolite2asn: Optional[bool] + geolite2country: Optional[bool] + geolite2city: Optional[bool] + customconf: Optional[bool] + + +# m_config_delete_blobs = api.model( +# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} +# ) + +class Config(BaseModel): + meta: Meta = {} + documents: ConfigDocuments = {} + blobs: ConfigBlobs = {} + delete_documents: ConfigDocuments = {} + delete_blobs: ConfigDeleteBlobs = {} + + +# m_config = api.model( +# "Config", +# { +# "meta": fields.Nested(m_meta, default={}), +# "documents": fields.Nested(m_config_documents, default={}), +# "blobs": fields.Nested(m_config_blobs, default={}), +# "delete_documents": fields.Nested(m_config_documents, default={}), +# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), +# }, +# ) + +class Edit(BaseModel): + path: str + value: str + + +# m_edit = api.model( +# "Edit", +# { +# "path": fields.String(required=True), +# "value": fields.String(required=True), +# }, +# ) + +class BasicEntry(BaseModel): + id: str + name: str + description: Optional[str] + + +# m_basic_entry = api.model( +# "Basic Document Entry", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# }, +# ) + +### Publish + +class Bucket(BaseModel): + name: str + url: str + + +# m_bucket = api.model( +# "Bucket", +# { +# "name": fields.String(required=True), +# "url": fields.String(required=True), +# }, +# ) + +### Git push & pull + +class GitUrl(BaseModel): + giturl: str + + +# m_giturl = api.model( +# "GitUrl", +# { +# "giturl": fields.String(required=True), +# }, +# ) + +### Db +class DB(BaseModel): + pass + + +# m_db = api.model("db", {}) + + +### Document Schema validation + + +def validateJson(json_data, schema_type): + try: + validate(instance=json_data, schema=schema_type_map[schema_type]) + except jsonschema.exceptions.ValidationError as err: + print(str(err)) + return False, str(err) + return True, "" + + +### DB Schema validation + + +def validateDbJson(json_data, schema): + try: + validate(instance=json_data, schema=schema) + except jsonschema.exceptions.ValidationError as err: + print(str(err)) + return False + return True + + +### Set git actor according to config & defined HTTP headers + + +def get_gitactor(request): + email, username = "", "" + email_header = app.options.get("trusted_email_header", None) + if email_header: + email = request.headers.get(email_header, "") + username_header = app.options.get("trusted_username_header", None) + if username_header: + username = request.headers.get(username_header, "") + return app.backend.prepare_actor(username, email) + + +base_path = Path(__file__).parent +# base_path = "/etc/curiefense/json/" +acl_profile_file_path = (base_path / "./json/acl-profile.schema").resolve() +with open(acl_profile_file_path) as json_file: + acl_profile_schema = json.load(json_file) +ratelimits_file_path = (base_path / "./json/rate-limits.schema").resolve() +with open(ratelimits_file_path) as json_file: + ratelimits_schema = json.load(json_file) +securitypolicies_file_path = (base_path / "./json/security-policies.schema").resolve() +with open(securitypolicies_file_path) as json_file: + securitypolicies_schema = json.load(json_file) +content_filter_profile_file_path = ( + base_path / "./json/content-filter-profile.schema" +).resolve() +with open(content_filter_profile_file_path) as json_file: + content_filter_profile_schema = json.load(json_file) +globalfilters_file_path = (base_path / "./json/global-filters.schema").resolve() +with open(globalfilters_file_path) as json_file: + globalfilters_schema = json.load(json_file) +flowcontrol_file_path = (base_path / "./json/flow-control.schema").resolve() +with open(flowcontrol_file_path) as json_file: + flowcontrol_schema = json.load(json_file) +content_filter_rule_file_path = ( + base_path / "./json/content-filter-rule.schema" +).resolve() +with open(content_filter_rule_file_path) as json_file: + content_filter_rule_schema = json.load(json_file) +action_file_path = (base_path / "./json/action.schema").resolve() +with open(action_file_path) as json_file: + action_schema = json.load(json_file) +virtualtag_file_path = (base_path / "./json/virtual-tags.schema").resolve() +with open(virtualtag_file_path) as json_file: + virtual_tags_schema = json.load(json_file) +custom_file_path = (base_path / "./json/custom.schema").resolve() +with open(custom_file_path) as json_file: + custom_schema = json.load(json_file) +schema_type_map = { + "ratelimits": ratelimits_schema, + "securitypolicies": securitypolicies_schema, + "contentfilterprofiles": content_filter_profile_schema, + "aclprofiles": acl_profile_schema, + "globalfilters": globalfilters_schema, + "flowcontrol": flowcontrol_schema, + "contentfilterrules": content_filter_rule_schema, + "actions": action_schema, + "virtualtags": virtual_tags_schema, + "custom": custom_schema, +} + + + + + +if __name__ == '__main__': + print("hi") From f4f6e6ba7bbaccefa3720296551685aa6359fb0c Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Wed, 16 Nov 2022 21:00:59 +0200 Subject: [PATCH 02/55] end of day backup --- .../curieconf/confserver/v3/fast_api.py | 703 +++++++++++++++++- 1 file changed, 699 insertions(+), 4 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 0dcff4755..68f820566 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1,8 +1,9 @@ import datetime import typing +from enum import Enum from typing import Optional, List, Union -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, HTTPException from pydantic import BaseModel, Field, validator import random # needed for generating a random number for an API import uvicorn # optional if you run it directly from terminal @@ -11,9 +12,23 @@ # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type jsonschema.Draft4Validator = jsonschema.Draft3Validator -from curieconf import utils -from curieconf.utils import cloud -from curieconf.confserver import app +# from curieconf import utils +# from curieconf.utils import cloud +# from curieconf.confserver import app + + +# TODO: TEMP DEFINITIONS +import os + +app = FastAPI() +app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") +options = {} +val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) +if val: + options["trusted_username_header"] = val +val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) +if val: + options["trusted_email_header"] = val import requests from jsonschema import validate @@ -745,7 +760,687 @@ def get_gitactor(request): } +class Tags(Enum): + congifs = "configs" + db = "db" + tools = "tools" + + +################ +### CONFIGS ### +################ + +@app.get("/configs/", tags=[Tags.congifs], response_model=Meta) +async def configs_get(): + """Get the detailed list of existing configurations""" + return app.backend.configs_list() + + +@app.post("/configs/", tags=[Tags.congifs]) +async def configs_post(config: Config, request: Request): + """Create a new configuration""" + data = dict(config) + return app.backend.configs_create(data, get_gitactor(request)) + + +# +# @ns_configs.route("/") +# class Configs(Resource): +# # @ns_configs.marshal_list_with(m_meta, skip_none=True) +# # def get(self): +# # "Get the detailed list of existing configurations" +# # return current_app.backend.configs_list() +# +# @ns_configs.expect(m_config, validate=True) +# def post(self): +# "Create a new configuration" +# data = request.json +# return current_app.backend.configs_create(data, get_gitactor()) + +@app.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +async def config_get(config: str): + """Retrieve a complete configuration""" + return app.backend.configs_get(config) + + +@app.post("/configs/{config}/", tags=[Tags.congifs]) +async def config_post(config: str, m_config: Config, request: Request): + "Create a new configuration. Configuration name in URL overrides configuration in POST data" + data = dict(m_config) + return app.backend.configs_create(data, config, get_gitactor(request)) + + +@app.put("/configs/{config}/", tags=[Tags.congifs]) +async def config_put(config: str, meta: Meta, request: Request): + """Update an existing configuration""" + data = dict(meta) + return app.backend.configs_update(config, data, get_gitactor(request)) + + +@app.delete("/configs/{config}/", tags=[Tags.congifs]) +async def config_delete(config: str): + """Delete a configuration""" + return app.backend.configs_delete(config) + + +# @ns_configs.route("//") +# class Config(Resource): +# # @ns_configs.marshal_with(m_config, skip_none=True) +# # def get(self, config): +# # "Retrieve a complete configuration" +# # return current_app.backend.configs_get(config) +# +# # @ns_configs.expect(m_config, validate=True) +# # def post(self, config): +# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" +# # data = request.json +# # return current_app.backend.configs_create(data, config, get_gitactor()) +# +# # @ns_configs.expect(m_meta, validate=True) +# # def put(self, config): +# # "Update an existing configuration" +# # data = request.json +# # return current_app.backend.configs_update(config, data, get_gitactor()) +# +# def delete(self, config): +# "Delete a configuration" +# return current_app.backend.configs_delete(config) + + +@app.post("/configs/{config}/clone/", tags=[Tags.congifs]) +async def config_clone_post(config: str, meta: Meta): + """Clone a configuration. New name is provided in POST data""" + data = dict(meta) + return app.backend.configs_clone(config, data) + + +# @ns_configs.route("//clone/") +# class ConfigClone(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config): +# "Clone a configuration. New name is provided in POST data" +# data = request.json +# return current_app.backend.configs_clone(config, data) +# + +@app.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) +async def config_clone_name_post(config: str, new_name: str, meta: Meta): + """Clone a configuration. New name is provided URL""" + data = dict(meta) + return app.backend.configs_clone(config, data, new_name) + + +# @ns_configs.route("//clone//") +# class ConfigCloneName(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config, new_name): +# "Clone a configuration. New name is provided URL" +# data = request.json +# return current_app.backend.configs_clone(config, data, new_name) + + +@app.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def config_list_version_get(config: str): + """Get all versions of a given configuration""" + return app.backend.configs_list_versions(config) + + +# @ns_configs.route("//v/") +# class ConfigListVersion(Resource): +# @ns_configs.marshal_with(m_version_log, skip_none=True) +# def get(self, config): +# "Get all versions of a given configuration" +# return current_app.backend.configs_list_versions(config) + + +@app.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +async def config_version_get(config: str, version: str): + """Retrieve a specific version of a configuration""" + return app.backend.configs_get(config, version) + + +# @ns_configs.route("//v//") +# class ConfigVersion(Resource): +# def get(self, config, version): +# "Retrieve a specific version of a configuration" +# return current_app.backend.configs_get(config, version) + +@app.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +async def config_revert_put(config: str, version: str, request: Request): + """Create a new version for a configuration from an old version""" + return app.backend.configs_revert(config, version, get_gitactor(request)) + + +# @ns_configs.route("//v//revert/") +# class ConfigRevert(Resource): +# def put(self, config, version): +# "Create a new version for a configuration from an old version" +# return current_app.backend.configs_revert(config, version, get_gitactor()) + + +############# +### Blobs ### +############# + + +@app.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +async def blobs_resource_get(config: str): + """Retrieve the list of available blobs""" + res = app.backend.blobs_list(config) + return res + + +# @ns_configs.route("//b/") +# class BlobsResource(Resource): +# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of available blobs" +# res = current_app.backend.blobs_list(config) +# return res + +@app.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +async def blob_resource_get(config: str, blob: str): + """Retrieve a blob""" + return app.backend.blobs_get(config, blob) + + +@app.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_create( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@app.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_update( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@app.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_get(config: str, blob: str, request: Request): + """Delete a blob""" + return app.backend.blobs_delete(config, blob, get_gitactor(request)) + + +# +# @ns_configs.route("//b//") +# class BlobResource(Resource): +# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) +# # def get(self, config, blob): +# # "Retrieve a blob" +# # return current_app.backend.blobs_get(config, blob) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def post(self, config, blob): +# # "Create a new blob" +# # return current_app.backend.blobs_create( +# # config, blob, request.json, get_gitactor() +# # ) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def put(self, config, blob): +# # "Replace a blob with new data" +# # return current_app.backend.blobs_update( +# # config, blob, request.json, get_gitactor() +# # ) +# +# def delete(self, config, blob): +# "Delete a blob" +# return current_app.backend.blobs_delete(config, blob, get_gitactor()) + + +@app.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def blob_list_version_resource_get(config: str, blob: str): + "Retrieve the list of versions of a given blob" + res = app.backend.blobs_list_versions(config, blob) + return res + + +# @ns_configs.route("//b//v/") +# class BlobListVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob): +# "Retrieve the list of versions of a given blob" +# res = current_app.backend.blobs_list_versions(config, blob) +# return res + + +@app.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +async def blob_version_resource_get(config: str, blob: str, version: str): + """Retrieve the given version of a blob""" + return app.backend.blobs_get(config, blob, version) + + +# @ns_configs.route("//b//v//") +# class BlobVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob, version): +# "Retrieve the given version of a blob" +# return current_app.backend.blobs_get(config, blob, version) + +@app.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): + """Create a new version for a blob from an old version""" + return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + +# +# @ns_configs.route("//b//v//revert/") +# class BlobRevertResource(Resource): +# def put(self, config, blob, version): +# "Create a new version for a blob from an old version" +# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) + + +################# +### DOCUMENTS ### +################# + +@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model= DocumentListEntry) +async def document_resource(config:str): + """Retrieve the list of existing documents in this configuration""" + res = app.backend.documents_list(config) + return res + +# +# @ns_configs.route("//d/") +# class DocumentsResource(Resource): +# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of existing documents in this configuration" +# res = current_app.backend.documents_list(config) +# return res + + +@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model= DocumentMask ) +async def get(config: str, document: str): + pass +@app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def post(config: str, document: str, basic_entry: List[BasicEntry]): + "Create a new complete document" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = dict[basic_entry] + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched: \n" + err) + res = current_app.backend.documents_create( + config, document, data, get_gitactor() + ) + return res + + + +@ns_configs.route("//d//") +class DocumentResource(Resource): + @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) + def get(self, config, document): + "Get a complete document" + if document not in models: + + abort(404, "document does not exist") + res = current_app.backend.documents_get(config, document) + return marshal(res, models[document], skip_none=True) + + @ns_configs.expect([m_basic_entry], validate=True) + def post(self, config, document): + "Create a new complete document" + if document not in models: + abort(404, "document does not exist") + data = marshal(request.json, models[document], skip_none=True) + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched: \n" + err) + res = current_app.backend.documents_create( + config, document, data, get_gitactor() + ) + return res + + @ns_configs.expect([m_basic_entry], validate=True) + def put(self, config, document): + "Update an existing document" + if document not in models: + abort(404, "document does not exist") + data = marshal(request.json, models[document], skip_none=True) + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = current_app.backend.documents_update( + config, document, data, get_gitactor() + ) + return res + + def delete(self, config, document): + "Delete/empty a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_delete(config, document, get_gitactor()) + return res + + +@ns_configs.route("//d//v/") +class DocumentListVersionResource(Resource): + def get(self, config, document): + "Retrieve the existing versions of a given document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_list_versions(config, document) + return marshal(res, m_version_log, skip_none=True) + + +@ns_configs.route("//d//v//") +class DocumentVersionResource(Resource): + def get(self, config, document, version): + "Get a given version of a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_get(config, document, version) + return marshal(res, models[document], skip_none=True) + + +@ns_configs.route("//d//v//revert/") +class DocumentRevertResource(Resource): + def put(self, config, document, version): + "Create a new version for a document from an old version" + return current_app.backend.documents_revert( + config, document, version, get_gitactor() + ) + + +############### +### ENTRIES ### +############### + + +@ns_configs.route("//d//e/") +class EntriesResource(Resource): + def get(self, config, document): + "Retrieve the list of entries in a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_list(config, document) + return res # XXX: marshal + + @ns_configs.expect(m_basic_entry, validate=True) + def post(self, config, document): + "Create an entry in a document" + if document not in models: + abort(404, "document does not exist") + isValid, err = validateJson(request.json, document) + if isValid: + data = marshal(request.json, models[document], skip_none=True) + res = current_app.backend.entries_create( + config, document, data, get_gitactor() + ) + return res + else: + abort(500, "schema mismatched: \n" + err) + + +@ns_configs.route("//d//e//") +class EntryResource(Resource): + def get(self, config, document, entry): + "Retrieve an entry from a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_get(config, document, entry) + return marshal(res, models[document], skip_none=True) + + @ns_configs.expect(m_basic_entry, validate=True) + def put(self, config, document, entry): + "Update an entry in a document" + if document not in models: + abort(404, "document does not exist") + isValid, err = validateJson(request.json, document) + if isValid: + data = marshal(request.json, models[document], skip_none=True) + res = current_app.backend.entries_update( + config, document, entry, data, get_gitactor() + ) + return res + else: + abort(500, "schema mismatched: \n" + err) + + def delete(self, config, document, entry): + "Delete an entry from a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_delete( + config, document, entry, get_gitactor() + ) + return res + + +@ns_configs.route("//d//e//v/") +class EntryListVersionResource(Resource): + def get(self, config, document, entry): + "Get the list of existing versions of a given entry in a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_list_versions(config, document, entry) + return marshal(res, m_version_log, skip_none=True) + + +@ns_configs.route( + "//d//e//v//" +) +class EntryVersionResource(Resource): + def get(self, config, document, entry, version): + "Get a given version of a document entry" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_get(config, document, entry, version) + return marshal(res, models[document], skip_none=True) + + +################ +### Database ### +################ + + +@ns_db.route("/") +class DbResource(Resource): + def get(self): + "Get the list of existing namespaces" + return current_app.backend.ns_list() + + +@ns_db.route("/v/") +class DbQueryResource(Resource): + def get(self): + "List all existing versions of namespaces" + return current_app.backend.ns_list_versions() + + +@ns_db.route("//") +class NSResource(Resource): + def get(self, nsname): + "Get a complete namespace" + try: + return current_app.backend.ns_get(nsname, version=None) + except KeyError: + abort(404, "namespace [%s] does not exist" % nsname) + + @ns_db.expect(m_db, validate=True) + def post(self, nsname): + "Create a non-existing namespace from data" + try: + return current_app.backend.ns_create(nsname, request.json, get_gitactor()) + except Exception: + abort(409, "namespace [%s] already exists" % nsname) + + @ns_db.expect(m_db, validate=True) + def put(self, nsname): + "Merge data into a namespace" + return current_app.backend.ns_update(nsname, request.json, get_gitactor()) + + def delete(self, nsname): + "Delete an existing namespace" + try: + return current_app.backend.ns_delete(nsname, get_gitactor()) + except KeyError: + abort(409, "namespace [%s] does not exist" % nsname) + + +@ns_db.route("//v//") +class NSVersionResource(Resource): + def get(self, nsname, version): + "Get a given version of a namespace" + return current_app.backend.ns_get(nsname, version) + + +@ns_db.route("//v//revert/") +class NSVersionResource(Resource): + def put(self, nsname, version): + "Create a new version for a namespace from an old version" + try: + return current_app.backend.ns_revert(nsname, version, get_gitactor()) + except KeyError: + abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@ns_db.route("//q/") +class NSQueryResource(Resource): + def post(self, nsname): + "Run a JSON query on the namespace and returns the results" + return current_app.backend.ns_query(nsname, request.json) + + +@ns_db.route("//k/") +class KeysResource(Resource): + def get(self, nsname): + "List all keys of a given namespace" + return current_app.backend.key_list(nsname) + + +@ns_db.route("//k//v/") +class KeysListVersionsResource(Resource): + def get(self, nsname, key): + "Get all versions of a given key in namespace" + return current_app.backend.key_list_versions(nsname, key) + + +@ns_db.route("//k//") +class KeyResource(Resource): + def get(self, nsname, key): + "Retrieve a given key's value from a given namespace" + return current_app.backend.key_get(nsname, key) + + def put(self, nsname, key): + "Create or update the value of a key" + # check if "reblaze/k/" exists in system/schema-validation + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = current_app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(request.json, schema) + if isValid is False: + abort(500, "schema mismatched") + return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) + + def delete(self, nsname, key): + "Delete a key" + return current_app.backend.key_delete(nsname, key, get_gitactor()) + + +############# +### Tools ### +############# + + +req_fetch_parser = reqparse.RequestParser() +req_fetch_parser.add_argument("url", location="args", help="url to retrieve") + + +@ns_tools.route("/fetch") +class FetchResource(Resource): + @ns_tools.expect(req_fetch_parser, validate=True) + def get(self): + "Fetch an URL" + args = req_fetch_parser.parse_args() + try: + r = requests.get(args.url) + except Exception as e: + abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) + return make_response(r.content) + + +@ns_tools.route("/publish//") +@ns_tools.route("/publish//v//") +class PublishResource(Resource): + @ns_tools.expect([m_bucket], validate=True) + def put(self, config, version=None): + "Push configuration to s3 buckets" + conf = current_app.backend.configs_get(config, version) + ok = True + status = [] + if type(request.json) is not list: + abort(400, "body must be a list") + for bucket in request.json: + logs = [] + try: + cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) + except Exception as e: + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} + ) + return make_response({"ok": ok, "status": status}) + + +@ns_tools.route("/gitpush/") +class GitPushResource(Resource): + @ns_tools.expect([m_giturl], validate=True) + def put(self): + "Push git configuration to remote git repositories" + ok = True + status = [] + for giturl in request.json: + try: + current_app.backend.gitpush(giturl["giturl"]) + except Exception as e: + msg = repr(e) + s = False + else: + msg = "ok" + s = True + status.append({"url": giturl["giturl"], "ok": s, "message": msg}) + return make_response({"ok": ok, "status": status}) + + +@ns_tools.route("/gitfetch/") +class GitFetchResource(Resource): + @ns_tools.expect(m_giturl, validate=True) + def put(self): + "Fetch git configuration from specified remote repository" + ok = True + try: + current_app.backend.gitfetch(request.json["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return make_response({"ok": ok, "status": msg}) if __name__ == '__main__': From 6c7ae88842a5105c27b9b12d37e71fe89251bffb Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 12:05:23 +0200 Subject: [PATCH 03/55] backup --- .../curieconf/confserver/v3/fast_api.py | 222 +++++++++++------- 1 file changed, 142 insertions(+), 80 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 68f820566..1396ff2c8 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1027,6 +1027,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request """Create a new version for a blob from an old version""" return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + # # @ns_configs.route("//b//v//revert/") # class BlobRevertResource(Resource): @@ -1039,12 +1040,13 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ### DOCUMENTS ### ################# -@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model= DocumentListEntry) -async def document_resource(config:str): +@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +async def document_resource(config: str): """Retrieve the list of existing documents in this configuration""" res = app.backend.documents_list(config) return res + # # @ns_configs.route("//d/") # class DocumentsResource(Resource): @@ -1055,104 +1057,164 @@ async def document_resource(config:str): # return res -@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model= DocumentMask ) -async def get(config: str, document: str): - pass +@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +async def document_resource_get(config: str, document: str): + """Get a complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + res = app.backend.documents_get(config, document) + res = {key: res[key] for key in list(models[document].__fields__.keys())} + return res + + +async def _filter(data, keys): + return {key: data[key] for key in keys} + @app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def post(config: str, document: str, basic_entry: List[BasicEntry]): - "Create a new complete document" +async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Create a new complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - data = dict[basic_entry] - for entry in request.json: - isValid, err = validateJson(entry, document) + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() + raise HTTPException(500, "schema mismatched: \n" + err) + res = app.backend.documents_create( + config, document, data, get_gitactor(request) ) return res +@app.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Update an existing document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") -@ns_configs.route("//d//") -class DocumentResource(Resource): - @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) - def get(self, config, document): - "Get a complete document" - if document not in models: - - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document) - return marshal(res, models[document], skip_none=True) + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = app.backend.documents_update( + config, document, data, get_gitactor(request) + ) + return res - @ns_configs.expect([m_basic_entry], validate=True) - def post(self, config, document): - "Create a new complete document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() - ) - return res - @ns_configs.expect([m_basic_entry], validate=True) - def put(self, config, document): - "Update an existing document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = current_app.backend.documents_update( - config, document, data, get_gitactor() - ) - return res +@app.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_delete(config: str, document: str, request: Request): + """Delete/empty a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_delete(config, document, get_gitactor(request)) + return res - def delete(self, config, document): - "Delete/empty a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_delete(config, document, get_gitactor()) - return res +# @ns_configs.route("//d//") +# class DocumentResource(Resource): +# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) +# # def get(self, config, document): +# # "Get a complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_get(config, document) +# # return marshal(res, models[document], skip_none=True) +# # +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def post(self, config, document): +# # "Create a new complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched: \n" + err) +# # res = current_app.backend.documents_create( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def put(self, config, document): +# # "Update an existing document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) +# # res = current_app.backend.documents_update( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # def delete(self, config, document): +# # "Delete/empty a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_delete(config, document, get_gitactor()) +# # return res -@ns_configs.route("//d//v/") -class DocumentListVersionResource(Resource): - def get(self, config, document): - "Retrieve the existing versions of a given document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_list_versions(config, document) - return marshal(res, m_version_log, skip_none=True) +@app.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) +async def document_list_version_resource_get(config: str, document: str): + """Retrieve the existing versions of a given document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_list_versions(config, document) + res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res -@ns_configs.route("//d//v//") -class DocumentVersionResource(Resource): - def get(self, config, document, version): - "Get a given version of a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document, version) - return marshal(res, models[document], skip_none=True) +# +# @ns_configs.route("//d//v/") +# class DocumentListVersionResource(Resource): +# def get(self, config, document): +# "Retrieve the existing versions of a given document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_list_versions(config, document) +# return marshal(res, m_version_log, skip_none=True) + + +@app.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) +async def document_version_resource_get(config: str, document: str, version: str): + """Get a given version of a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_get(config, document, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route("//d//v//") +# class DocumentVersionResource(Resource): +# def get(self, config, document, version): +# "Get a given version of a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_get(config, document, version) +# return marshal(res, models[document], skip_none=True) + +@app.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) +async def document_revert_resource_put(config: str, document: str, version: str, request: Request): + """Create a new version for a document from an old version""" + return app.backend.documents_revert( + config, document, version, get_gitactor(request) + ) -@ns_configs.route("//d//v//revert/") -class DocumentRevertResource(Resource): - def put(self, config, document, version): - "Create a new version for a document from an old version" - return current_app.backend.documents_revert( - config, document, version, get_gitactor() - ) +# @ns_configs.route("//d//v//revert/") +# class DocumentRevertResource(Resource): +# def put(self, config, document, version): +# "Create a new version for a document from an old version" +# return current_app.backend.documents_revert( +# config, document, version, get_gitactor() +# ) ############### From 9d2d1529995bae2b40ae171cfebffac490600754 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 13:50:16 +0200 Subject: [PATCH 04/55] configs done --- .../curieconf/confserver/v3/fast_api.py | 231 ++++++++++++------ 1 file changed, 156 insertions(+), 75 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 1396ff2c8..e22aee840 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1208,6 +1208,7 @@ async def document_revert_resource_put(config: str, document: str, version: str, config, document, version, get_gitactor(request) ) + # @ns_configs.route("//d//v//revert/") # class DocumentRevertResource(Resource): # def put(self, config, document, version): @@ -1221,86 +1222,166 @@ async def document_revert_resource_put(config: str, document: str, version: str, ### ENTRIES ### ############### +@app.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_get(config: str, document: str): + """Retrieve the list of entries in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list(config, document) + return res # XXX: marshal -@ns_configs.route("//d//e/") -class EntriesResource(Resource): - def get(self, config, document): - "Retrieve the list of entries in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list(config, document) - return res # XXX: marshal - - @ns_configs.expect(m_basic_entry, validate=True) - def post(self, config, document): - "Create an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_create( - config, document, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - -@ns_configs.route("//d//e//") -class EntryResource(Resource): - def get(self, config, document, entry): - "Retrieve an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect(m_basic_entry, validate=True) - def put(self, config, document, entry): - "Update an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_update( - config, document, entry, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - def delete(self, config, document, entry): - "Delete an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_delete( - config, document, entry, get_gitactor() + +@app.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): + "Create an entry in a document" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + res = app.backend.entries_create( + config, document, data, get_gitactor(request) ) return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +# @ns_configs.route("//d//e/") +# class EntriesResource(Resource): +# # def get(self, config, document): +# # "Retrieve the list of entries in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_list(config, document) +# # return res # XXX: marshal +# +# @ns_configs.expect(m_basic_entry, validate=True) +# def post(self, config, document): +# "Create an entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# isValid, err = validateJson(request.json, document) +# if isValid: +# data = marshal(request.json, models[document], skip_none=True) +# res = current_app.backend.entries_create( +# config, document, data, get_gitactor() +# ) +# return res +# else: +# abort(500, "schema mismatched: \n" + err) + + +@app.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_get(config: str, document: str, entry: str): + """Retrieve an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry) + return {key: res for key in list(models[document].__fields__.keys())} + + +@app.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): + """Update an entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + + res = app.backend.entries_update( + config, document, entry, data, get_gitactor(request) + ) + return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +@app.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): + """Delete an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_delete( + config, document, entry, get_gitactor(request) + ) + return res -@ns_configs.route("//d//e//v/") -class EntryListVersionResource(Resource): - def get(self, config, document, entry): - "Get the list of existing versions of a given entry in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list_versions(config, document, entry) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route( - "//d//e//v//" -) -class EntryVersionResource(Resource): - def get(self, config, document, entry, version): - "Get a given version of a document entry" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry, version) - return marshal(res, models[document], skip_none=True) +# @ns_configs.route("//d//e//") +# class EntryResource(Resource): +# # def get(self, config, document, entry): +# # "Retrieve an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_get(config, document, entry) +# # return marshal(res, models[document], skip_none=True) +# +# # @ns_configs.expect(m_basic_entry, validate=True) +# # def put(self, config, document, entry): +# # "Update an entry in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # isValid, err = validateJson(request.json, document) +# # if isValid: +# # data = marshal(request.json, models[document], skip_none=True) +# # res = current_app.backend.entries_update( +# # config, document, entry, data, get_gitactor() +# # ) +# # return res +# # else: +# # abort(500, "schema mismatched: \n" + err) +# +# # def delete(self, config, document, entry): +# # "Delete an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_delete( +# # config, document, entry, get_gitactor() +# # ) +# # return res + + +@app.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) +async def entry_list_version_resource_get(config: str, document: str, entry: str): + """Get the list of existing versions of a given entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list_versions(config, document, entry) + return {key: res[key] for key in list(VersionLog.__fields__.keys())} + + +# +# @ns_configs.route("//d//e//v/") +# class EntryListVersionResource(Resource): +# def get(self, config, document, entry): +# "Get the list of existing versions of a given entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_list_versions(config, document, entry) +# return marshal(res, m_version_log, skip_none=True) + + +@app.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) +async def entry_version_resource_get(config: str, document: str, entry: str, version: str): + """Get a given version of a document entry""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route( +# "//d//e//v//" +# ) +# class EntryVersionResource(Resource): +# def get(self, config, document, entry, version): +# "Get a given version of a document entry" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_get(config, document, entry, version) +# return marshal(res, models[document], skip_none=True) ################ From 5f628a2ab3c6781f8226b3d9cca378b60ec87ed7 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 15:30:54 +0200 Subject: [PATCH 05/55] db done --- .../curieconf/confserver/v3/fast_api.py | 310 ++++++++++++------ 1 file changed, 212 insertions(+), 98 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index e22aee840..e19c0c6ef 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1388,116 +1388,230 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver ### Database ### ################ +@app.get("/db/", tags=[Tags.db]) +async def db_resource_get(): + """Get the list of existing namespaces""" + return app.backend.ns_list() -@ns_db.route("/") -class DbResource(Resource): - def get(self): - "Get the list of existing namespaces" - return current_app.backend.ns_list() +# @ns_db.route("/") +# class DbResource(Resource): +# def get(self): +# "Get the list of existing namespaces" +# return current_app.backend.ns_list() -@ns_db.route("/v/") -class DbQueryResource(Resource): - def get(self): - "List all existing versions of namespaces" - return current_app.backend.ns_list_versions() +@app.get("/db/v/", tags=[Tags.db]) +async def db_query_resource_get(): + """List all existing versions of namespaces""" + return app.backend.ns_list_versions() -@ns_db.route("//") -class NSResource(Resource): - def get(self, nsname): - "Get a complete namespace" - try: - return current_app.backend.ns_get(nsname, version=None) - except KeyError: - abort(404, "namespace [%s] does not exist" % nsname) - @ns_db.expect(m_db, validate=True) - def post(self, nsname): - "Create a non-existing namespace from data" - try: - return current_app.backend.ns_create(nsname, request.json, get_gitactor()) - except Exception: - abort(409, "namespace [%s] already exists" % nsname) +# +# @ns_db.route("/v/") +# class DbQueryResource(Resource): +# def get(self): +# "List all existing versions of namespaces" +# return current_app.backend.ns_list_versions() - @ns_db.expect(m_db, validate=True) - def put(self, nsname): - "Merge data into a namespace" - return current_app.backend.ns_update(nsname, request.json, get_gitactor()) - def delete(self, nsname): - "Delete an existing namespace" - try: - return current_app.backend.ns_delete(nsname, get_gitactor()) - except KeyError: - abort(409, "namespace [%s] does not exist" % nsname) +@app.get("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_get(nsname: str): + """Get a complete namespace""" + try: + return app.backend.ns_get(nsname, version=None) + except KeyError: + raise HTTPException(404, "namespace [%s] does not exist" % nsname) -@ns_db.route("//v//") -class NSVersionResource(Resource): - def get(self, nsname, version): - "Get a given version of a namespace" - return current_app.backend.ns_get(nsname, version) +@app.post("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_post(nsname: str, db: DB, request: Request): + """Create a non-existing namespace from data""" + try: + return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + except Exception: + raise HTTPException(409, "namespace [%s] already exists" % nsname) -@ns_db.route("//v//revert/") -class NSVersionResource(Resource): - def put(self, nsname, version): - "Create a new version for a namespace from an old version" - try: - return current_app.backend.ns_revert(nsname, version, get_gitactor()) - except KeyError: - abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@ns_db.route("//q/") -class NSQueryResource(Resource): - def post(self, nsname): - "Run a JSON query on the namespace and returns the results" - return current_app.backend.ns_query(nsname, request.json) - - -@ns_db.route("//k/") -class KeysResource(Resource): - def get(self, nsname): - "List all keys of a given namespace" - return current_app.backend.key_list(nsname) - - -@ns_db.route("//k//v/") -class KeysListVersionsResource(Resource): - def get(self, nsname, key): - "Get all versions of a given key in namespace" - return current_app.backend.key_list_versions(nsname, key) - - -@ns_db.route("//k//") -class KeyResource(Resource): - def get(self, nsname, key): - "Retrieve a given key's value from a given namespace" - return current_app.backend.key_get(nsname, key) - - def put(self, nsname, key): - "Create or update the value of a key" - # check if "reblaze/k/" exists in system/schema-validation - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = current_app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(request.json, schema) - if isValid is False: - abort(500, "schema mismatched") - return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) - - def delete(self, nsname, key): - "Delete a key" - return current_app.backend.key_delete(nsname, key, get_gitactor()) +@app.put("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, db: DB, request: Request): + """Merge data into a namespace""" + return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + + +@app.delete("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, request: Request): + """Delete an existing namespace""" + try: + return app.backend.ns_delete(nsname, get_gitactor(request)) + except KeyError: + raise HTTPException(409, "namespace [%s] does not exist" % nsname) + + +# @ns_db.route("//") +# class NSResource(Resource): +# # def get(self, nsname): +# # "Get a complete namespace" +# # try: +# # return current_app.backend.ns_get(nsname, version=None) +# # except KeyError: +# # abort(404, "namespace [%s] does not exist" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def post(self, nsname): +# # "Create a non-existing namespace from data" +# # try: +# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) +# # except Exception: +# # abort(409, "namespace [%s] already exists" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def put(self, nsname): +# # "Merge data into a namespace" +# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) +# +# # def delete(self, nsname): +# # "Delete an existing namespace" +# # try: +# # return current_app.backend.ns_delete(nsname, get_gitactor()) +# # except KeyError: +# # abort(409, "namespace [%s] does not exist" % nsname) + + +@app.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +async def ns_version_resource_get(nsname: str, version: str): + """Get a given version of a namespace""" + return app.backend.ns_get(nsname, version) + + +# @ns_db.route("//v//") +# class NSVersionResource(Resource): +# def get(self, nsname, version): +# "Get a given version of a namespace" +# return current_app.backend.ns_get(nsname, version) + + +@app.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) +async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): + """Create a new version for a namespace from an old version""" + try: + return app.backend.ns_revert(nsname, version, get_gitactor(request)) + except KeyError: + raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +# +# @ns_db.route("//v//revert/") +# class NSVersionResource(Resource): +# def put(self, nsname, version): +# "Create a new version for a namespace from an old version" +# try: +# return current_app.backend.ns_revert(nsname, version, get_gitactor()) +# except KeyError: +# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@app.post("/db/{nsname}/q/", tags=[Tags.db]) +async def ns_query_resource_post(nsname: str, request: Request): + """Run a JSON query on the namespace and returns the results""" + req_json = await request.json() + return app.backend.ns_query(nsname, req_json) + + +# @ns_db.route("//q/") +# class NSQueryResource(Resource): +# def post(self, nsname): +# "Run a JSON query on the namespace and returns the results" +# return current_app.backend.ns_query(nsname, request.json) +# + + +@app.get("/db/{nsname}/k/", tags=[Tags.db]) +async def keys_resource_get(nsname: str): + """List all keys of a given namespace""" + return app.backend.key_list(nsname) + + +# @ns_db.route("//k/") +# class KeysResource(Resource): +# def get(self, nsname): +# "List all keys of a given namespace" +# return current_app.backend.key_list(nsname) + +@app.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) +async def keys_list_versions_resource_get(nsname: str, key: str): + """Get all versions of a given key in namespace""" + return app.backend.key_list_versions(nsname, key) + + +# @ns_db.route("//k//v/") +# class KeysListVersionsResource(Resource): +# def get(self, nsname, key): +# "Get all versions of a given key in namespace" +# return current_app.backend.key_list_versions(nsname, key) +# + +@app.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_get(nsname: str, key: str): + """Retrieve a given key's value from a given namespace""" + return app.backend.key_get(nsname, key) + + +@app.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_put(nsname: str, key: str, request: Request): + """Create or update the value of a key""" + # check if "reblaze/k/" exists in system/schema-validation + req_json = await request.json() + + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(req_json, schema) + if isValid is False: + raise HTTPException(500, "schema mismatched") + return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + + +@app.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_delete(nsname: str, key: str, request: Request): + """Delete a key""" + return app.backend.key_delete(nsname, key, get_gitactor(request)) + + +# @ns_db.route("//k//") +# class KeyResource(Resource): +# # def get(self, nsname, key): +# # "Retrieve a given key's value from a given namespace" +# # return current_app.backend.key_get(nsname, key) +# +# # def put(self, nsname, key): +# # "Create or update the value of a key" +# # # check if "reblaze/k/" exists in system/schema-validation +# # if nsname != "system": +# # keyName = nsname + "/k/" + key +# # schemas = current_app.backend.key_get("system", "schema-validation") +# # schema = None +# # # find schema if exists and validate the json input +# # for item in schemas.items(): +# # if item[0] == keyName: +# # schema = item[1] +# # break +# # if schema: +# # isValid = validateDbJson(request.json, schema) +# # if isValid is False: +# # abort(500, "schema mismatched") +# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) +# +# # def delete(self, nsname, key): +# # "Delete a key" +# # return current_app.backend.key_delete(nsname, key, get_gitactor()) ############# From 12d73be4652b0a9e8f1cd3be7d65345d008d9d25 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 18:59:07 +0200 Subject: [PATCH 06/55] initial mirroring complete --- .../curieconf/confserver/v3/fast_api.py | 207 ++++++++++++------ 1 file changed, 138 insertions(+), 69 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index e19c0c6ef..076af7992 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -13,7 +13,7 @@ jsonschema.Draft4Validator = jsonschema.Draft3Validator # from curieconf import utils -# from curieconf.utils import cloud +from curieconf.utils import cloud # from curieconf.confserver import app @@ -1623,81 +1623,150 @@ async def key_resource_delete(nsname: str, key: str, request: Request): req_fetch_parser.add_argument("url", location="args", help="url to retrieve") -@ns_tools.route("/fetch") -class FetchResource(Resource): - @ns_tools.expect(req_fetch_parser, validate=True) - def get(self): - "Fetch an URL" - args = req_fetch_parser.parse_args() +@app.get("/tools/fetch", tags=[Tags.tools]) +async def fetch_resource_get(url: str): + """Fetch an URL""" + try: + r = requests.get(url) + except Exception as e: + raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) + return r.content + + +# @ns_tools.route("/fetch") +# class FetchResource(Resource): +# @ns_tools.expect(req_fetch_parser, validate=True) +# def get(self): +# "Fetch an URL" +# args = req_fetch_parser.parse_args() +# try: +# r = requests.get(args.url) +# except Exception as e: +# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) +# return make_response(r.content) + + +@app.put("/tools/publish/{config}/", tags=[Tags.tools]) +@app.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) +async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): + """Push configuration to s3 buckets""" + conf = app.backend.configs_get(config, version) + ok = True + status = [] + req_json = await request.json() + if type(req_json) is not list: + raise HTTPException(400, "body must be a list") + for bucket in buckets: + logs = [] try: - r = requests.get(args.url) + cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) except Exception as e: - abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) - return make_response(r.content) - - -@ns_tools.route("/publish//") -@ns_tools.route("/publish//v//") -class PublishResource(Resource): - @ns_tools.expect([m_bucket], validate=True) - def put(self, config, version=None): - "Push configuration to s3 buckets" - conf = current_app.backend.configs_get(config, version) - ok = True - status = [] - if type(request.json) is not list: - abort(400, "body must be a list") - for bucket in request.json: - logs = [] - try: - cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitpush/") -class GitPushResource(Resource): - @ns_tools.expect([m_giturl], validate=True) - def put(self): - "Push git configuration to remote git repositories" - ok = True - status = [] - for giturl in request.json: - try: - current_app.backend.gitpush(giturl["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": giturl["giturl"], "ok": s, "message": msg}) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitfetch/") -class GitFetchResource(Resource): - @ns_tools.expect(m_giturl, validate=True) - def put(self): - "Fetch git configuration from specified remote repository" - ok = True + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + ) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/publish//") +# @ns_tools.route("/publish//v//") +# class PublishResource(Resource): +# @ns_tools.expect([m_bucket], validate=True) +# def put(self, config, version=None): +# "Push configuration to s3 buckets" +# conf = current_app.backend.configs_get(config, version) +# ok = True +# status = [] +# if type(request.json) is not list: +# abort(400, "body must be a list") +# for bucket in request.json: +# logs = [] +# try: +# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) +# except Exception as e: +# ok = False +# s = False +# msg = repr(e) +# else: +# s = True +# msg = "ok" +# status.append( +# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} +# ) +# return make_response({"ok": ok, "status": status}) + + +@app.put("/tools/gitpush/", tags=[Tags.tools]) +async def git_push_resource_put(git_urls: List[GitUrl]): + """Push git configuration to remote git repositories""" + ok = True + status = [] + for giturl in git_urls: try: - current_app.backend.gitfetch(request.json["giturl"]) + app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: - ok = False msg = repr(e) + s = False else: msg = "ok" - return make_response({"ok": ok, "status": msg}) + s = True + status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/gitpush/") +# class GitPushResource(Resource): +# @ns_tools.expect([m_giturl], validate=True) +# def put(self): +# "Push git configuration to remote git repositories" +# ok = True +# status = [] +# for giturl in request.json: +# try: +# current_app.backend.gitpush(giturl["giturl"]) +# except Exception as e: +# msg = repr(e) +# s = False +# else: +# msg = "ok" +# s = True +# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) +# return make_response({"ok": ok, "status": status}) + + +@app.put("/tools/gitfetch/", tags=[Tags.tools]) +async def git_fetch_resource_put(giturl: GitUrl): + """Fetch git configuration from specified remote repository""" + ok = True + try: + app.backend.gitfetch(dict(giturl)["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return {"ok": ok, "status": msg} + + +# @ns_tools.route("/gitfetch/") +# class GitFetchResource(Resource): +# @ns_tools.expect(m_giturl, validate=True) +# def put(self): +# "Fetch git configuration from specified remote repository" +# ok = True +# try: +# current_app.backend.gitfetch(request.json["giturl"]) +# except Exception as e: +# ok = False +# msg = repr(e) +# else: +# msg = "ok" +# return make_response({"ok": ok, "status": msg}) if __name__ == '__main__': From f2d7cb3be998060ba578dc3a4496e390b00472f6 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Sun, 20 Nov 2022 19:33:46 +0200 Subject: [PATCH 07/55] running server, pre checks --- .../server/curieconf/confserver/__init__.py | 42 +- .../server/curieconf/confserver/v3/api.py | 2369 +++++++++++------ curiefense/curieconf/server/setup.py | 1 + 3 files changed, 1579 insertions(+), 833 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 13c981103..0dbe9486f 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -1,26 +1,15 @@ #! /usr/bin/env python3 import os -import flask -from flask import Flask, current_app from .backend import Backends - -from flask_cors import CORS +import uvicorn +from curieconf.confserver.v3 import api from prometheus_flask_exporter import PrometheusMetrics +from fastapi import FastAPI -## Import all versions -from .v1 import api as api_v1 -from .v2 import api as api_v2 -from .v3 import api as api_v3 - -app = Flask(__name__) - -CORS(app, resources={r"/*": {"origins": "*"}}) - -app.register_blueprint(api_v1.api_bp, url_prefix="/api/v1") -app.register_blueprint(api_v2.api_bp, url_prefix="/api/v2") -app.register_blueprint(api_v3.api_bp, url_prefix="/api/v3") +app = FastAPI() +app.include_router(api.router) def drop_into_pdb(app, exception): @@ -60,19 +49,16 @@ def main(args=None): options = parser.parse_args(args) - if options.pdb: - flask.got_request_exception.connect(drop_into_pdb) - - metrics = PrometheusMetrics(app) + #TODO - find replacements for got_request_exception and prometheus_flask_exporter + # if options.pdb: + # flask.got_request_exception.connect(drop_into_pdb) + # metrics = PrometheusMetrics(app) try: - with app.app_context(): - current_app.backend = Backends.get_backend(app, options.dbpath) - current_app.options = options.__dict__ - app.run(debug=options.debug, host=options.host, port=options.port) + app.backend = Backends.get_backend(app, options.dbpath) + app.options = options.__dict__ + uvicorn.run(app, host=options.host, port=options.port) + + # app.run(debug=options.debug, host=options.host, port=options.port) finally: pass - - -if __name__ == "__main__": - main() diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index ed1770883..8cc9c0781 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1,25 +1,45 @@ +import datetime +import typing +from enum import Enum +from typing import Optional, List, Union + +from fastapi import FastAPI, Request, HTTPException, APIRouter +from pydantic import BaseModel, Field, validator +import random # needed for generating a random number for an API +import uvicorn # optional if you run it directly from terminal import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type jsonschema.Draft4Validator = jsonschema.Draft3Validator -from flask import Blueprint, request, current_app, abort, make_response -from flask_restx import Resource, Api, fields, marshal, reqparse -from curieconf import utils +# from curieconf import utils from curieconf.utils import cloud +# from curieconf.confserver import app + + +# TODO: TEMP DEFINITIONS +import os + +router = APIRouter() +options = {} +val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) +if val: + options["trusted_username_header"] = val +val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) +if val: + options["trusted_email_header"] = val + import requests from jsonschema import validate from pathlib import Path import json +# api_bp = Blueprint("api_v3", __name__) +# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") -api_bp = Blueprint("api_v3", __name__) -api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - - -ns_configs = api.namespace("configs", description="Configurations") -ns_db = api.namespace("db", description="Database") -ns_tools = api.namespace("tools", description="Tools") +# ns_configs = api.namespace("configs", description="Configurations") +# ns_db = api.namespace("db", description="Database") +# ns_tools = api.namespace("tools", description="Tools") ############## @@ -28,371 +48,628 @@ ### Models for documents +anyTypeUnion = Union[int, float, bool, object, list, None] +anyOp = Optional[object] +anyType = ["number", "string", "boolean", "object", "array", "null"] -class AnyType(fields.Raw): - __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] +# class AnyType(fields.Raw): +# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] # limit -m_threshold = api.model( - "Rate Limit Threshold", - { - "limit": fields.Integer(required=True), - "action": fields.String(required=True), - }, -) - -m_limit = api.model( - "Rate Limit", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "global": fields.Boolean(required=True), - "active": fields.Boolean(required=True), - "timeframe": fields.Integer(required=True), - "thresholds": fields.List(fields.Nested(m_threshold)), - "include": fields.Raw(required=True), - "exclude": fields.Raw(required=True), - "key": AnyType(required=True), - "pairwith": fields.Raw(required=True), - "tags": fields.List(fields.String()), - }, -) +class Threshold(BaseModel): + limit: int + action: str + + +# m_threshold = api.model( +# "Rate Limit Threshold", +# { +# "limit": fields.Integer(required=True), +# "action": fields.String(required=True), +# }, +# ) + +class Limit(BaseModel): + id: str + name: str + description: Optional[str] + _global: bool = Field(alias="global") + active: bool + timeframe: int + thresholds: List[Threshold] + include: typing.Any + exclude: typing.Any + key: anyTypeUnion + pairwith: typing.Any + tags: List[str] + + +# m_limit = api.model( +# "Rate Limit", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "global": fields.Boolean(required=True), +# "active": fields.Boolean(required=True), +# "timeframe": fields.Integer(required=True), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "include": fields.Raw(required=True), +# "exclude": fields.Raw(required=True), +# "key": AnyType(required=True), +# "pairwith": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# }, +# ) # securitypolicy - -m_secprofilemap = api.model( - "Security Profile Map", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "match": fields.String(required=True), - "acl_profile": fields.String(required=True), - "acl_active": fields.Boolean(required=True), - "content_filter_profile": fields.String(required=True), - "content_filter_active": fields.Boolean(required=True), - "limit_ids": fields.List(fields.Raw()), - }, -) - -m_map = api.model( - "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -) - -m_securitypolicy = api.model( - "Security Policy", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "tags": fields.List(fields.String()), - "match": fields.String(required=True), - "session": AnyType(), - "session_ids": AnyType(), - "map": fields.List(fields.Nested(m_secprofilemap)), - }, -) +class SecProfileMap(BaseModel): + id: str + name: str + description: str + match: str + acl_profile: str + acl_active: bool + content_filter_profile: str + content_filter_active: bool + limit_ids: Optional[list] + + +# m_secprofilemap = api.model( +# "Security Profile Map", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.String(required=True), +# "acl_profile": fields.String(required=True), +# "acl_active": fields.Boolean(required=True), +# "content_filter_profile": fields.String(required=True), +# "content_filter_active": fields.Boolean(required=True), +# "limit_ids": fields.List(fields.Raw()), +# }, +# ) + +# TODO = deprecated? +# m_map = api.model( +# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} +# ) + +class SecurityPolicy(BaseModel): + id: str + name: str + description: str + tags: List[str] + match: str + session: anyTypeUnion + session_ids: anyTypeUnion + map: List[SecProfileMap] + + +# m_securitypolicy = api.model( +# "Security Policy", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String()), +# "match": fields.String(required=True), +# "session": AnyType(), +# "session_ids": AnyType(), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# }, +# ) # content filter rule -m_contentfilterrule = api.model( - "Content Filter Rule", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "msg": fields.String(required=True), - "operand": fields.String(required=True), - "severity": fields.Integer(required=True), - "certainity": fields.Integer(required=True), - "category": fields.String(required=True), - "subcategory": fields.String(required=True), - "risk": fields.Integer(required=True), - "tags": fields.List(fields.String()), - "description": fields.String(), - }, -) +class ContentFilterRule(BaseModel): + id: str + name: str + msg: str + operand: str + severity: int + certainity: int + category: str + subcategory: str + risk: int + tags: Optional[List[str]] + description: Optional[str] + + +# m_contentfilterrule = api.model( +# "Content Filter Rule", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "msg": fields.String(required=True), +# "operand": fields.String(required=True), +# "severity": fields.Integer(required=True), +# "certainity": fields.Integer(required=True), +# "category": fields.String(required=True), +# "subcategory": fields.String(required=True), +# "risk": fields.Integer(required=True), +# "tags": fields.List(fields.String()), +# "description": fields.String(), +# }, +# ) # content filter profile - -m_contentfilterprofile = api.model( - "Content Filter Profile", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "ignore_alphanum": fields.Boolean(required=True), - "args": fields.Raw(required=True), - "headers": fields.Raw(required=True), - "cookies": fields.Raw(required=True), - "path": fields.Raw(required=True), - "allsections": fields.Raw(), - "decoding": fields.Raw(required=True), - "masking_seed": fields.String(required=True), - "content_type": fields.List(fields.String()), - "active": fields.List(fields.String()), - "report": fields.List(fields.String()), - "ignore": fields.List(fields.String()), - "tags": fields.List(fields.String()), - "action": fields.String(), - "ignore_body": fields.Boolean(required=True), - }, -) +class ContentFilterProfile(BaseModel): + id: str + name: str + description: Optional[str] + ignore_alphanum: bool + args: typing.Any + headers: typing.Any + cookies: typing.Any + path: typing.Any + allsections: typing.Any + decoding: typing.Any + masking_seed: str + content_type: Optional[List[str]] + active: Optional[List[str]] + report: Optional[List[str]] + ignore: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + ignore_body: bool + + +# m_contentfilterprofile = api.model( +# "Content Filter Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "ignore_alphanum": fields.Boolean(required=True), +# "args": fields.Raw(required=True), +# "headers": fields.Raw(required=True), +# "cookies": fields.Raw(required=True), +# "path": fields.Raw(required=True), +# "allsections": fields.Raw(), +# "decoding": fields.Raw(required=True), +# "masking_seed": fields.String(required=True), +# "content_type": fields.List(fields.String()), +# "active": fields.List(fields.String()), +# "report": fields.List(fields.String()), +# "ignore": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# "ignore_body": fields.Boolean(required=True), +# }, +# ) # aclprofile - -m_aclprofile = api.model( - "ACL Profile", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "allow": fields.List(fields.String()), - "allow_bot": fields.List(fields.String()), - "deny_bot": fields.List(fields.String()), - "passthrough": fields.List(fields.String()), - "deny": fields.List(fields.String()), - "force_deny": fields.List(fields.String()), - "tags": fields.List(fields.String()), - "action": fields.String(), - }, -) +class ACLProfile(BaseModel): + id: str + name: str + description: Optional[str] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + + +# m_aclprofile = api.model( +# "ACL Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# }, +# ) # Global Filter - -m_globalfilter = api.model( - "Global Filter", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "source": fields.String(required=True), - "mdate": fields.String(required=True), - "description": fields.String(), - "active": fields.Boolean(required=True), - "action": fields.Raw(required=True), - "tags": fields.List(fields.String()), - "rule": AnyType(), - }, -) +class GlobalFilter(BaseModel): + id: str + name: str + source: str + mdate: str + description: str + active: bool + action: typing.Any + tags: Optional[List[str]] + rule: anyTypeUnion + + +# m_glbalfilter = api.model( +# "Global Filter", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "source": fields.String(required=True), +# "mdate": fields.String(required=True), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# "action": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# "rule": AnyType(), +# }, +# ) # Flow Control -m_flowcontrol = api.model( - "Flow Control", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "timeframe": fields.Integer(required=True), - "key": fields.List(fields.Raw(required=True)), - "sequence": fields.List(fields.Raw(required=True)), - "tags": fields.List(fields.String()), - "include": fields.List(fields.String()), - "exclude": fields.List(fields.String()), - "description": fields.String(), - "active": fields.Boolean(required=True), - }, -) +class FlowControl(BaseModel): + id: str + name: str + timeframe: int + key: List[typing.Any] + sequence: List[typing.Any] + tags: Optional[List[str]] + include: Optional[List[str]] + exclude: Optional[List[str]] + description: Optional[str] + active: bool + + +# +# m_flowcontrol = api.model( +# "Flow Control", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "timeframe": fields.Integer(required=True), +# "key": fields.List(fields.Raw(required=True)), +# "sequence": fields.List(fields.Raw(required=True)), +# "tags": fields.List(fields.String()), +# "include": fields.List(fields.String()), +# "exclude": fields.List(fields.String()), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# }, +# ) # Action -m_action = api.model( - "Action", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "tags": fields.List(fields.String(required=True)), - "params": fields.Raw(), - "type": fields.String(required=True), - }, -) +class Action(BaseModel): + id: str + name: str + description: Optional[str] + tags: List[str] + params: typing.Any + type: str + + +# m_action = api.model( +# "Action", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String(required=True)), +# "params": fields.Raw(), +# "type": fields.String(required=True), +# }, +# ) # Virtual Tag - -m_virtualtag = api.model( - "Virtual Tag", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "match": fields.List(fields.Raw(required=True)), - }, -) +class VirtualTag(BaseModel): + id: str + name: str + description: Optional[str] + match: List[typing.Any] + + +# +# m_virtualtag = api.model( +# "Virtual Tag", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.List(fields.Raw(required=True)), +# }, +# ) # custom +class Custom(BaseModel): + id: str + name: str + -m_custom = api.model( - "Custom", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "*": fields.Wildcard(fields.Raw()), - }, -) +# m_custom = api.model( +# "Custom", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) ### mapping from doc name to model models = { - "ratelimits": m_limit, - "securitypolicies": m_securitypolicy, - "contentfilterrules": m_contentfilterrule, - "contentfilterprofiles": m_contentfilterprofile, - "aclprofiles": m_aclprofile, - "globalfilters": m_globalfilter, - "flowcontrol": m_flowcontrol, - "actions": m_action, - "virtualtags": m_custom, - "custom": m_custom, + "ratelimits": Limit, + "securitypolicies": SecurityPolicy, + "contentfilterrules": ContentFilterRule, + "contentfilterprofiles": ContentFilterProfile, + "aclprofiles": ACLProfile, + "globalfilters": GlobalFilter, + "flowcontrol": FlowControl, + "actions": Action, + "virtualtags": Custom, + "custom": Custom, } -### Other models -m_document_mask = api.model( - "Mask for document", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(required=True), - "map": fields.List(fields.Nested(m_secprofilemap)), - "include": fields.Wildcard(fields.Raw()), - "exclude": fields.Wildcard(fields.Raw()), - "tags": fields.List(fields.String()), - "active": fields.Wildcard(fields.Raw()), - "action": fields.Raw(), - "sequence": fields.List(fields.Raw()), - "timeframe": fields.Integer(), - "thresholds": fields.List(fields.Nested(m_threshold)), - "pairwith": fields.Raw(), - "content_type": fields.List(fields.String()), - "params": fields.Raw(), - "decoding": fields.Raw(), - "category": fields.String(), - "subcategory": fields.String(), - "risk": fields.Integer(), - "allow": fields.List(fields.String()), - "allow_bot": fields.List(fields.String()), - "deny_bot": fields.List(fields.String()), - "passthrough": fields.List(fields.String()), - "deny": fields.List(fields.String()), - "force_deny": fields.List(fields.String()), - "match": fields.String(), - "type": fields.String(), - "*": fields.Wildcard(fields.Raw()), - }, -) - -m_version_log = api.model( - "Version log", - { - "version": fields.String(), - "date": fields.DateTime(dt_format="iso8601"), - "*": fields.Wildcard(fields.Raw()), - }, -) - -m_meta = api.model( - "Meta", - { - "id": fields.String(required=True), - "description": fields.String(required=True), - "date": fields.DateTime(), - "logs": fields.List(fields.Nested(m_version_log), default=[]), - "version": fields.String(), - }, -) - -m_blob_entry = api.model( - "Blob Entry", - { - "format": fields.String(required=True), - "blob": AnyType(), - }, -) - -m_blob_list_entry = api.model( - "Blob ListEntry", - { - "name": fields.String(), - }, -) - -m_document_list_entry = api.model( - "Document ListEntry", - { - "name": fields.String(), - "entries": fields.Integer(), - }, -) - - -m_config_documents = api.model( - "Config Documents", - {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -) - -m_config_blobs = api.model( - "Config Blobs", - {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -) - -m_config_delete_blobs = api.model( - "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -) - -m_config = api.model( - "Config", - { - "meta": fields.Nested(m_meta, default={}), - "documents": fields.Nested(m_config_documents, default={}), - "blobs": fields.Nested(m_config_blobs, default={}), - "delete_documents": fields.Nested(m_config_documents, default={}), - "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), - }, -) - -m_edit = api.model( - "Edit", - { - "path": fields.String(required=True), - "value": fields.String(required=True), - }, -) - -m_basic_entry = api.model( - "Basic Document Entry", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - }, -) +### Other models +class DocumentMask(BaseModel): + id: str + name: str + description: str + map: Optional[List[SecProfileMap]] + include: Optional[List[typing.Any]] + exclude: Optional[List[typing.Any]] + tags: Optional[List[str]] + active: Optional[List[typing.Any]] + action: typing.Any + sequence: Optional[List[typing.Any]] + timeframe: Optional[int] + thresholds: Optional[List[Threshold]] + pairwith: typing.Any + content_type: Optional[List[str]] + params: typing.Any + decoding: typing.Any + category: Optional[str] + subcategory: Optional[str] + risk: Optional[int] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + match: Optional[str] = "j" + _type: Optional[str] = Field(alias="type") + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# m_document_mask = api.model( +# "Mask for document", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(required=True), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# "include": fields.Wildcard(fields.Raw()), +# "exclude": fields.Wildcard(fields.Raw()), +# "tags": fields.List(fields.String()), +# "active": fields.Wildcard(fields.Raw()), +# "action": fields.Raw(), +# "sequence": fields.List(fields.Raw()), +# "timeframe": fields.Integer(), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "pairwith": fields.Raw(), +# "content_type": fields.List(fields.String()), +# "params": fields.Raw(), +# "decoding": fields.Raw(), +# "category": fields.String(), +# "subcategory": fields.String(), +# "risk": fields.Integer(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "match": fields.String(), +# "type": fields.String(), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class VersionLog(BaseModel): + version: Optional[str] + # TODO - dt_format="iso8601" + date: Optional[datetime.datetime] + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# +# m_version_log = api.model( +# "Version log", +# { +# "version": fields.String(), +# "date": fields.DateTime(dt_format="iso8601"), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class Meta(BaseModel): + id: str + description: str + date: Optional[datetime.datetime] + logs: Optional[List[VersionLog]] = [] + version: Optional[str] + + +# m_meta = api.model( +# "Meta", +# { +# "id": fields.String(required=True), +# "description": fields.String(required=True), +# "date": fields.DateTime(), +# "logs": fields.List(fields.Nested(m_version_log), default=[]), +# "version": fields.String(), +# }, +# ) + +class BlobEntry(BaseModel): + format: str + blob: anyTypeUnion + + +# m_blob_entry = api.model( +# "Blob Entry", +# { +# "format": fields.String(required=True), +# "blob": AnyType(), +# }, +# ) + +class BlobListEntry(BaseModel): + name: Optional[str] + + +# m_blob_list_entry = api.model( +# "Blob ListEntry", +# { +# "name": fields.String(), +# }, +# ) + +class DocumentListEntry(BaseModel): + name: Optional[str] + entries: Optional[int] + + +# m_document_list_entry = api.model( +# "Document ListEntry", +# { +# "name": fields.String(), +# "entries": fields.Integer(), +# }, +# ) + +class ConfigDocuments(BaseModel): + ratelimits: Optional[List[models["ratelimits"]]] = [] + securitypolicies: Optional[List[models["securitypolicies"]]] = [] + contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] + contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] + aclprofiles: Optional[List[models["aclprofiles"]]] = [] + globalfilters: Optional[List[models["globalfilters"]]] = [] + flowcontrol: Optional[List[models["flowcontrol"]]] = [] + actions: Optional[List[models["actions"]]] = [] + virtualtags: Optional[List[models["virtualtags"]]] = [] + custom: Optional[List[models["custom"]]] = [] + + +# m_config_documents = api.model( +# "Config Documents", +# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, +# ) + + +class ConfigBlobs(BaseModel): + geolite2asn: Optional[List[Optional[BlobEntry]]] + geolite2country: Optional[List[Optional[BlobEntry]]] + geolite2city: Optional[List[Optional[BlobEntry]]] + customconf: Optional[List[Optional[BlobEntry]]] + + +# m_config_blobs = api.model( +# "Config Blobs", +# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, +# ) + +class ConfigDeleteBlobs(BaseModel): + geolite2asn: Optional[bool] + geolite2country: Optional[bool] + geolite2city: Optional[bool] + customconf: Optional[bool] + + +# m_config_delete_blobs = api.model( +# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} +# ) + +class Config(BaseModel): + meta: Meta = {} + documents: ConfigDocuments = {} + blobs: ConfigBlobs = {} + delete_documents: ConfigDocuments = {} + delete_blobs: ConfigDeleteBlobs = {} + + +# m_config = api.model( +# "Config", +# { +# "meta": fields.Nested(m_meta, default={}), +# "documents": fields.Nested(m_config_documents, default={}), +# "blobs": fields.Nested(m_config_blobs, default={}), +# "delete_documents": fields.Nested(m_config_documents, default={}), +# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), +# }, +# ) + +class Edit(BaseModel): + path: str + value: str + + +# m_edit = api.model( +# "Edit", +# { +# "path": fields.String(required=True), +# "value": fields.String(required=True), +# }, +# ) + +class BasicEntry(BaseModel): + id: str + name: str + description: Optional[str] + + +# m_basic_entry = api.model( +# "Basic Document Entry", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# }, +# ) ### Publish -m_bucket = api.model( - "Bucket", - { - "name": fields.String(required=True), - "url": fields.String(required=True), - }, -) +class Bucket(BaseModel): + name: str + url: str + + +# m_bucket = api.model( +# "Bucket", +# { +# "name": fields.String(required=True), +# "url": fields.String(required=True), +# }, +# ) ### Git push & pull -m_giturl = api.model( - "GitUrl", - { - "giturl": fields.String(required=True), - }, -) +class GitUrl(BaseModel): + giturl: str + +# m_giturl = api.model( +# "GitUrl", +# { +# "giturl": fields.String(required=True), +# }, +# ) ### Db +class DB(BaseModel): + pass + + +# m_db = api.model("db", {}) -m_db = api.model("db", {}) ### Document Schema validation @@ -421,15 +698,15 @@ def validateDbJson(json_data, schema): ### Set git actor according to config & defined HTTP headers -def get_gitactor(): +def get_gitactor(request): email, username = "", "" - email_header = current_app.options.get("trusted_email_header", None) + email_header = app.options.get("trusted_email_header", None) if email_header: email = request.headers.get(email_header, "") - username_header = current_app.options.get("trusted_username_header", None) + username_header = app.options.get("trusted_username_header", None) if username_header: username = request.headers.get(username_header, "") - return current_app.backend.prepare_actor(username, email) + return app.backend.prepare_actor(username, email) base_path = Path(__file__).parent @@ -444,7 +721,7 @@ def get_gitactor(): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -455,7 +732,7 @@ def get_gitactor(): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -468,8 +745,6 @@ def get_gitactor(): custom_file_path = (base_path / "./json/custom.schema").resolve() with open(custom_file_path) as json_file: custom_schema = json.load(json_file) - - schema_type_map = { "ratelimits": ratelimits_schema, "securitypolicies": securitypolicies_schema, @@ -484,87 +759,163 @@ def get_gitactor(): } +class Tags(Enum): + congifs = "configs" + db = "db" + tools = "tools" + + ################ ### CONFIGS ### ################ - -@ns_configs.route("/") -class Configs(Resource): - @ns_configs.marshal_list_with(m_meta, skip_none=True) - def get(self): - "Get the detailed list of existing configurations" - return current_app.backend.configs_list() - - @ns_configs.expect(m_config, validate=True) - def post(self): - "Create a new configuration" - data = request.json - return current_app.backend.configs_create(data, get_gitactor()) - - -@ns_configs.route("//") -class Config(Resource): - @ns_configs.marshal_with(m_config, skip_none=True) - def get(self, config): - "Retrieve a complete configuration" - return current_app.backend.configs_get(config) - - @ns_configs.expect(m_config, validate=True) - def post(self, config): - "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = request.json - return current_app.backend.configs_create(data, config, get_gitactor()) - - @ns_configs.expect(m_meta, validate=True) - def put(self, config): - "Update an existing configuration" - data = request.json - return current_app.backend.configs_update(config, data, get_gitactor()) - - def delete(self, config): - "Delete a configuration" - return current_app.backend.configs_delete(config) - - -@ns_configs.route("//clone/") -class ConfigClone(Resource): - @ns_configs.expect(m_meta, validate=True) - def post(self, config): - "Clone a configuration. New name is provided in POST data" - data = request.json - return current_app.backend.configs_clone(config, data) - - -@ns_configs.route("//clone//") -class ConfigCloneName(Resource): - @ns_configs.expect(m_meta, validate=True) - def post(self, config, new_name): - "Clone a configuration. New name is provided URL" - data = request.json - return current_app.backend.configs_clone(config, data, new_name) - - -@ns_configs.route("//v/") -class ConfigListVersion(Resource): - @ns_configs.marshal_with(m_version_log, skip_none=True) - def get(self, config): - "Get all versions of a given configuration" - return current_app.backend.configs_list_versions(config) - - -@ns_configs.route("//v//") -class ConfigVersion(Resource): - def get(self, config, version): - "Retrieve a specific version of a configuration" - return current_app.backend.configs_get(config, version) - - -@ns_configs.route("//v//revert/") -class ConfigRevert(Resource): - def put(self, config, version): - "Create a new version for a configuration from an old version" - return current_app.backend.configs_revert(config, version, get_gitactor()) +@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta]) +async def configs_get(request: Request): + """Get the detailed list of existing configurations""" + res = request.app.backend.configs_list() + return res + + +@router.post("/configs/", tags=[Tags.congifs]) +async def configs_post(config: Config, request: Request): + """Create a new configuration""" + data = dict(config) + return request.app.backend.configs_create(data, get_gitactor(request)) + + +# +# @ns_configs.route("/") +# class Configs(Resource): +# # @ns_configs.marshal_list_with(m_meta, skip_none=True) +# # def get(self): +# # "Get the detailed list of existing configurations" +# # return current_app.backend.configs_list() +# +# @ns_configs.expect(m_config, validate=True) +# def post(self): +# "Create a new configuration" +# data = request.json +# return current_app.backend.configs_create(data, get_gitactor()) + +@router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +async def config_get(config: str): + """Retrieve a complete configuration""" + return app.backend.configs_get(config) + + +@router.post("/configs/{config}/", tags=[Tags.congifs]) +async def config_post(config: str, m_config: Config, request: Request): + "Create a new configuration. Configuration name in URL overrides configuration in POST data" + data = dict(m_config) + return app.backend.configs_create(data, config, get_gitactor(request)) + + +@router.put("/configs/{config}/", tags=[Tags.congifs]) +async def config_put(config: str, meta: Meta, request: Request): + """Update an existing configuration""" + data = dict(meta) + return app.backend.configs_update(config, data, get_gitactor(request)) + + +@router.delete("/configs/{config}/", tags=[Tags.congifs]) +async def config_delete(config: str): + """Delete a configuration""" + return app.backend.configs_delete(config) + + +# @ns_configs.route("//") +# class Config(Resource): +# # @ns_configs.marshal_with(m_config, skip_none=True) +# # def get(self, config): +# # "Retrieve a complete configuration" +# # return current_app.backend.configs_get(config) +# +# # @ns_configs.expect(m_config, validate=True) +# # def post(self, config): +# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" +# # data = request.json +# # return current_app.backend.configs_create(data, config, get_gitactor()) +# +# # @ns_configs.expect(m_meta, validate=True) +# # def put(self, config): +# # "Update an existing configuration" +# # data = request.json +# # return current_app.backend.configs_update(config, data, get_gitactor()) +# +# def delete(self, config): +# "Delete a configuration" +# return current_app.backend.configs_delete(config) + + +@router.post("/configs/{config}/clone/", tags=[Tags.congifs]) +async def config_clone_post(config: str, meta: Meta): + """Clone a configuration. New name is provided in POST data""" + data = dict(meta) + return app.backend.configs_clone(config, data) + + +# @ns_configs.route("//clone/") +# class ConfigClone(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config): +# "Clone a configuration. New name is provided in POST data" +# data = request.json +# return current_app.backend.configs_clone(config, data) +# + +@router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) +async def config_clone_name_post(config: str, new_name: str, meta: Meta): + """Clone a configuration. New name is provided URL""" + data = dict(meta) + return app.backend.configs_clone(config, data, new_name) + + +# @ns_configs.route("//clone//") +# class ConfigCloneName(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config, new_name): +# "Clone a configuration. New name is provided URL" +# data = request.json +# return current_app.backend.configs_clone(config, data, new_name) + + +@router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def config_list_version_get(config: str): + """Get all versions of a given configuration""" + return app.backend.configs_list_versions(config) + + +# @ns_configs.route("//v/") +# class ConfigListVersion(Resource): +# @ns_configs.marshal_with(m_version_log, skip_none=True) +# def get(self, config): +# "Get all versions of a given configuration" +# return current_app.backend.configs_list_versions(config) + + +@router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +async def config_version_get(config: str, version: str): + """Retrieve a specific version of a configuration""" + return app.backend.configs_get(config, version) + + +# @ns_configs.route("//v//") +# class ConfigVersion(Resource): +# def get(self, config, version): +# "Retrieve a specific version of a configuration" +# return current_app.backend.configs_get(config, version) + +@router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +async def config_revert_put(config: str, version: str, request: Request): + """Create a new version for a configuration from an old version""" + return app.backend.configs_revert(config, version, get_gitactor(request)) + + +# @ns_configs.route("//v//revert/") +# class ConfigRevert(Resource): +# def put(self, config, version): +# "Create a new version for a configuration from an old version" +# return current_app.backend.configs_revert(config, version, get_gitactor()) ############# @@ -572,356 +923,695 @@ def put(self, config, version): ############# -@ns_configs.route("//b/") -class BlobsResource(Resource): - @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) - def get(self, config): - "Retrieve the list of available blobs" - res = current_app.backend.blobs_list(config) - return res - - -@ns_configs.route("//b//") -class BlobResource(Resource): - @ns_configs.marshal_with(m_blob_entry, skip_none=True) - def get(self, config, blob): - "Retrieve a blob" - return current_app.backend.blobs_get(config, blob) - - @ns_configs.expect(m_blob_entry, validate=True) - def post(self, config, blob): - "Create a new blob" - return current_app.backend.blobs_create( - config, blob, request.json, get_gitactor() - ) - - @ns_configs.expect(m_blob_entry, validate=True) - def put(self, config, blob): - "Replace a blob with new data" - return current_app.backend.blobs_update( - config, blob, request.json, get_gitactor() - ) - - def delete(self, config, blob): - "Delete a blob" - return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@ns_configs.route("//b//v/") -class BlobListVersionResource(Resource): - @ns_configs.marshal_list_with(m_version_log, skip_none=True) - def get(self, config, blob): - "Retrieve the list of versions of a given blob" - res = current_app.backend.blobs_list_versions(config, blob) - return res - - -@ns_configs.route("//b//v//") -class BlobVersionResource(Resource): - @ns_configs.marshal_list_with(m_version_log, skip_none=True) - def get(self, config, blob, version): - "Retrieve the given version of a blob" - return current_app.backend.blobs_get(config, blob, version) - - -@ns_configs.route("//b//v//revert/") -class BlobRevertResource(Resource): - def put(self, config, blob, version): - "Create a new version for a blob from an old version" - return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) +@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +async def blobs_resource_get(config: str): + """Retrieve the list of available blobs""" + res = app.backend.blobs_list(config) + return res + + +# @ns_configs.route("//b/") +# class BlobsResource(Resource): +# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of available blobs" +# res = current_app.backend.blobs_list(config) +# return res + +@router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +async def blob_resource_get(config: str, blob: str): + """Retrieve a blob""" + return app.backend.blobs_get(config, blob) + + +@router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_create( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_update( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_get(config: str, blob: str, request: Request): + """Delete a blob""" + return app.backend.blobs_delete(config, blob, get_gitactor(request)) + + +# +# @ns_configs.route("//b//") +# class BlobResource(Resource): +# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) +# # def get(self, config, blob): +# # "Retrieve a blob" +# # return current_app.backend.blobs_get(config, blob) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def post(self, config, blob): +# # "Create a new blob" +# # return current_app.backend.blobs_create( +# # config, blob, request.json, get_gitactor() +# # ) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def put(self, config, blob): +# # "Replace a blob with new data" +# # return current_app.backend.blobs_update( +# # config, blob, request.json, get_gitactor() +# # ) +# +# def delete(self, config, blob): +# "Delete a blob" +# return current_app.backend.blobs_delete(config, blob, get_gitactor()) + + +@router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def blob_list_version_resource_get(config: str, blob: str): + "Retrieve the list of versions of a given blob" + res = app.backend.blobs_list_versions(config, blob) + return res + + +# @ns_configs.route("//b//v/") +# class BlobListVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob): +# "Retrieve the list of versions of a given blob" +# res = current_app.backend.blobs_list_versions(config, blob) +# return res + + +@router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +async def blob_version_resource_get(config: str, blob: str, version: str): + """Retrieve the given version of a blob""" + return app.backend.blobs_get(config, blob, version) + + +# @ns_configs.route("//b//v//") +# class BlobVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob, version): +# "Retrieve the given version of a blob" +# return current_app.backend.blobs_get(config, blob, version) + +@router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): + """Create a new version for a blob from an old version""" + return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + + +# +# @ns_configs.route("//b//v//revert/") +# class BlobRevertResource(Resource): +# def put(self, config, blob, version): +# "Create a new version for a blob from an old version" +# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) ################# ### DOCUMENTS ### ################# - -@ns_configs.route("//d/") -class DocumentsResource(Resource): - @ns_configs.marshal_with(m_document_list_entry, skip_none=True) - def get(self, config): - "Retrieve the list of existing documents in this configuration" - res = current_app.backend.documents_list(config) - return res - - -@ns_configs.route("//d//") -class DocumentResource(Resource): - @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) - def get(self, config, document): - "Get a complete document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect([m_basic_entry], validate=True) - def post(self, config, document): - "Create a new complete document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() - ) - return res - - @ns_configs.expect([m_basic_entry], validate=True) - def put(self, config, document): - "Update an existing document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = current_app.backend.documents_update( - config, document, data, get_gitactor() - ) - return res - - def delete(self, config, document): - "Delete/empty a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_delete(config, document, get_gitactor()) - return res - - -@ns_configs.route("//d//v/") -class DocumentListVersionResource(Resource): - def get(self, config, document): - "Retrieve the existing versions of a given document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_list_versions(config, document) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route("//d//v//") -class DocumentVersionResource(Resource): - def get(self, config, document, version): - "Get a given version of a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document, version) - return marshal(res, models[document], skip_none=True) - - -@ns_configs.route("//d//v//revert/") -class DocumentRevertResource(Resource): - def put(self, config, document, version): - "Create a new version for a document from an old version" - return current_app.backend.documents_revert( - config, document, version, get_gitactor() - ) +@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +async def document_resource(config: str): + """Retrieve the list of existing documents in this configuration""" + res = app.backend.documents_list(config) + return res + + +# +# @ns_configs.route("//d/") +# class DocumentsResource(Resource): +# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of existing documents in this configuration" +# res = current_app.backend.documents_list(config) +# return res + + +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +async def document_resource_get(config: str, document: str): + """Get a complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + res = app.backend.documents_get(config, document) + res = {key: res[key] for key in list(models[document].__fields__.keys())} + return res + + +async def _filter(data, keys): + return {key: data[key] for key in keys} + + +@router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Create a new complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched: \n" + err) + res = app.backend.documents_create( + config, document, data, get_gitactor(request) + ) + return res + + +@router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Update an existing document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = app.backend.documents_update( + config, document, data, get_gitactor(request) + ) + return res + + +@router.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_delete(config: str, document: str, request: Request): + """Delete/empty a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_delete(config, document, get_gitactor(request)) + return res + + +# @ns_configs.route("//d//") +# class DocumentResource(Resource): +# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) +# # def get(self, config, document): +# # "Get a complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_get(config, document) +# # return marshal(res, models[document], skip_none=True) +# # +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def post(self, config, document): +# # "Create a new complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched: \n" + err) +# # res = current_app.backend.documents_create( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def put(self, config, document): +# # "Update an existing document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) +# # res = current_app.backend.documents_update( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # def delete(self, config, document): +# # "Delete/empty a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_delete(config, document, get_gitactor()) +# # return res + + +@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) +async def document_list_version_resource_get(config: str, document: str): + """Retrieve the existing versions of a given document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_list_versions(config, document) + res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res + + +# +# @ns_configs.route("//d//v/") +# class DocumentListVersionResource(Resource): +# def get(self, config, document): +# "Retrieve the existing versions of a given document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_list_versions(config, document) +# return marshal(res, m_version_log, skip_none=True) + + +@router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) +async def document_version_resource_get(config: str, document: str, version: str): + """Get a given version of a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_get(config, document, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route("//d//v//") +# class DocumentVersionResource(Resource): +# def get(self, config, document, version): +# "Get a given version of a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_get(config, document, version) +# return marshal(res, models[document], skip_none=True) + +@router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) +async def document_revert_resource_put(config: str, document: str, version: str, request: Request): + """Create a new version for a document from an old version""" + return app.backend.documents_revert( + config, document, version, get_gitactor(request) + ) + + +# @ns_configs.route("//d//v//revert/") +# class DocumentRevertResource(Resource): +# def put(self, config, document, version): +# "Create a new version for a document from an old version" +# return current_app.backend.documents_revert( +# config, document, version, get_gitactor() +# ) ############### ### ENTRIES ### ############### - -@ns_configs.route("//d//e/") -class EntriesResource(Resource): - def get(self, config, document): - "Retrieve the list of entries in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list(config, document) - return res # XXX: marshal - - @ns_configs.expect(m_basic_entry, validate=True) - def post(self, config, document): - "Create an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_create( - config, document, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - -@ns_configs.route("//d//e//") -class EntryResource(Resource): - def get(self, config, document, entry): - "Retrieve an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect(m_basic_entry, validate=True) - def put(self, config, document, entry): - "Update an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_update( - config, document, entry, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - def delete(self, config, document, entry): - "Delete an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_delete( - config, document, entry, get_gitactor() +@router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_get(config: str, document: str): + """Retrieve the list of entries in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list(config, document) + return res # XXX: marshal + + +@router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): + "Create an entry in a document" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + res = app.backend.entries_create( + config, document, data, get_gitactor(request) ) return res - - -@ns_configs.route("//d//e//v/") -class EntryListVersionResource(Resource): - def get(self, config, document, entry): - "Get the list of existing versions of a given entry in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list_versions(config, document, entry) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route( - "//d//e//v//" -) -class EntryVersionResource(Resource): - def get(self, config, document, entry, version): - "Get a given version of a document entry" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry, version) - return marshal(res, models[document], skip_none=True) + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +# @ns_configs.route("//d//e/") +# class EntriesResource(Resource): +# # def get(self, config, document): +# # "Retrieve the list of entries in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_list(config, document) +# # return res # XXX: marshal +# +# @ns_configs.expect(m_basic_entry, validate=True) +# def post(self, config, document): +# "Create an entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# isValid, err = validateJson(request.json, document) +# if isValid: +# data = marshal(request.json, models[document], skip_none=True) +# res = current_app.backend.entries_create( +# config, document, data, get_gitactor() +# ) +# return res +# else: +# abort(500, "schema mismatched: \n" + err) + + +@router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_get(config: str, document: str, entry: str): + """Retrieve an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry) + return {key: res for key in list(models[document].__fields__.keys())} + + +@router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): + """Update an entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + + res = app.backend.entries_update( + config, document, entry, data, get_gitactor(request) + ) + return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +@router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): + """Delete an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_delete( + config, document, entry, get_gitactor(request) + ) + return res + + +# @ns_configs.route("//d//e//") +# class EntryResource(Resource): +# # def get(self, config, document, entry): +# # "Retrieve an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_get(config, document, entry) +# # return marshal(res, models[document], skip_none=True) +# +# # @ns_configs.expect(m_basic_entry, validate=True) +# # def put(self, config, document, entry): +# # "Update an entry in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # isValid, err = validateJson(request.json, document) +# # if isValid: +# # data = marshal(request.json, models[document], skip_none=True) +# # res = current_app.backend.entries_update( +# # config, document, entry, data, get_gitactor() +# # ) +# # return res +# # else: +# # abort(500, "schema mismatched: \n" + err) +# +# # def delete(self, config, document, entry): +# # "Delete an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_delete( +# # config, document, entry, get_gitactor() +# # ) +# # return res + + +@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) +async def entry_list_version_resource_get(config: str, document: str, entry: str): + """Get the list of existing versions of a given entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list_versions(config, document, entry) + return {key: res[key] for key in list(VersionLog.__fields__.keys())} + + +# +# @ns_configs.route("//d//e//v/") +# class EntryListVersionResource(Resource): +# def get(self, config, document, entry): +# "Get the list of existing versions of a given entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_list_versions(config, document, entry) +# return marshal(res, m_version_log, skip_none=True) + + +@router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) +async def entry_version_resource_get(config: str, document: str, entry: str, version: str): + """Get a given version of a document entry""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route( +# "//d//e//v//" +# ) +# class EntryVersionResource(Resource): +# def get(self, config, document, entry, version): +# "Get a given version of a document entry" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_get(config, document, entry, version) +# return marshal(res, models[document], skip_none=True) ################ ### Database ### ################ +@router.get("/db/", tags=[Tags.db]) +async def db_resource_get(): + """Get the list of existing namespaces""" + return app.backend.ns_list() -@ns_db.route("/") -class DbResource(Resource): - def get(self): - "Get the list of existing namespaces" - return current_app.backend.ns_list() +# @ns_db.route("/") +# class DbResource(Resource): +# def get(self): +# "Get the list of existing namespaces" +# return current_app.backend.ns_list() -@ns_db.route("/v/") -class DbQueryResource(Resource): - def get(self): - "List all existing versions of namespaces" - return current_app.backend.ns_list_versions() +@router.get("/db/v/", tags=[Tags.db]) +async def db_query_resource_get(): + """List all existing versions of namespaces""" + return app.backend.ns_list_versions() -@ns_db.route("//") -class NSResource(Resource): - def get(self, nsname): - "Get a complete namespace" - try: - return current_app.backend.ns_get(nsname, version=None) - except KeyError: - abort(404, "namespace [%s] does not exist" % nsname) - @ns_db.expect(m_db, validate=True) - def post(self, nsname): - "Create a non-existing namespace from data" - try: - return current_app.backend.ns_create(nsname, request.json, get_gitactor()) - except Exception: - abort(409, "namespace [%s] already exists" % nsname) +# +# @ns_db.route("/v/") +# class DbQueryResource(Resource): +# def get(self): +# "List all existing versions of namespaces" +# return current_app.backend.ns_list_versions() - @ns_db.expect(m_db, validate=True) - def put(self, nsname): - "Merge data into a namespace" - return current_app.backend.ns_update(nsname, request.json, get_gitactor()) - def delete(self, nsname): - "Delete an existing namespace" - try: - return current_app.backend.ns_delete(nsname, get_gitactor()) - except KeyError: - abort(409, "namespace [%s] does not exist" % nsname) +@router.get("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_get(nsname: str): + """Get a complete namespace""" + try: + return app.backend.ns_get(nsname, version=None) + except KeyError: + raise HTTPException(404, "namespace [%s] does not exist" % nsname) -@ns_db.route("//v//") -class NSVersionResource(Resource): - def get(self, nsname, version): - "Get a given version of a namespace" - return current_app.backend.ns_get(nsname, version) +@router.post("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_post(nsname: str, db: DB, request: Request): + """Create a non-existing namespace from data""" + try: + return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + except Exception: + raise HTTPException(409, "namespace [%s] already exists" % nsname) -@ns_db.route("//v//revert/") -class NSVersionResource(Resource): - def put(self, nsname, version): - "Create a new version for a namespace from an old version" - try: - return current_app.backend.ns_revert(nsname, version, get_gitactor()) - except KeyError: - abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@ns_db.route("//q/") -class NSQueryResource(Resource): - def post(self, nsname): - "Run a JSON query on the namespace and returns the results" - return current_app.backend.ns_query(nsname, request.json) - - -@ns_db.route("//k/") -class KeysResource(Resource): - def get(self, nsname): - "List all keys of a given namespace" - return current_app.backend.key_list(nsname) - - -@ns_db.route("//k//v/") -class KeysListVersionsResource(Resource): - def get(self, nsname, key): - "Get all versions of a given key in namespace" - return current_app.backend.key_list_versions(nsname, key) - - -@ns_db.route("//k//") -class KeyResource(Resource): - def get(self, nsname, key): - "Retrieve a given key's value from a given namespace" - return current_app.backend.key_get(nsname, key) - - def put(self, nsname, key): - "Create or update the value of a key" - # check if "reblaze/k/" exists in system/schema-validation - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = current_app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(request.json, schema) - if isValid is False: - abort(500, "schema mismatched") - return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) - - def delete(self, nsname, key): - "Delete a key" - return current_app.backend.key_delete(nsname, key, get_gitactor()) +@router.put("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, db: DB, request: Request): + """Merge data into a namespace""" + return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + + +@router.delete("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, request: Request): + """Delete an existing namespace""" + try: + return app.backend.ns_delete(nsname, get_gitactor(request)) + except KeyError: + raise HTTPException(409, "namespace [%s] does not exist" % nsname) + + +# @ns_db.route("//") +# class NSResource(Resource): +# # def get(self, nsname): +# # "Get a complete namespace" +# # try: +# # return current_app.backend.ns_get(nsname, version=None) +# # except KeyError: +# # abort(404, "namespace [%s] does not exist" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def post(self, nsname): +# # "Create a non-existing namespace from data" +# # try: +# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) +# # except Exception: +# # abort(409, "namespace [%s] already exists" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def put(self, nsname): +# # "Merge data into a namespace" +# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) +# +# # def delete(self, nsname): +# # "Delete an existing namespace" +# # try: +# # return current_app.backend.ns_delete(nsname, get_gitactor()) +# # except KeyError: +# # abort(409, "namespace [%s] does not exist" % nsname) + + +@router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +async def ns_version_resource_get(nsname: str, version: str): + """Get a given version of a namespace""" + return app.backend.ns_get(nsname, version) + + +# @ns_db.route("//v//") +# class NSVersionResource(Resource): +# def get(self, nsname, version): +# "Get a given version of a namespace" +# return current_app.backend.ns_get(nsname, version) + + +@router.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) +async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): + """Create a new version for a namespace from an old version""" + try: + return app.backend.ns_revert(nsname, version, get_gitactor(request)) + except KeyError: + raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +# +# @ns_db.route("//v//revert/") +# class NSVersionResource(Resource): +# def put(self, nsname, version): +# "Create a new version for a namespace from an old version" +# try: +# return current_app.backend.ns_revert(nsname, version, get_gitactor()) +# except KeyError: +# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@router.post("/db/{nsname}/q/", tags=[Tags.db]) +async def ns_query_resource_post(nsname: str, request: Request): + """Run a JSON query on the namespace and returns the results""" + req_json = await request.json() + return app.backend.ns_query(nsname, req_json) + + +# @ns_db.route("//q/") +# class NSQueryResource(Resource): +# def post(self, nsname): +# "Run a JSON query on the namespace and returns the results" +# return current_app.backend.ns_query(nsname, request.json) +# + + +@router.get("/db/{nsname}/k/", tags=[Tags.db]) +async def keys_resource_get(nsname: str): + """List all keys of a given namespace""" + return app.backend.key_list(nsname) + + +# @ns_db.route("//k/") +# class KeysResource(Resource): +# def get(self, nsname): +# "List all keys of a given namespace" +# return current_app.backend.key_list(nsname) + +@router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) +async def keys_list_versions_resource_get(nsname: str, key: str): + """Get all versions of a given key in namespace""" + return app.backend.key_list_versions(nsname, key) + + +# @ns_db.route("//k//v/") +# class KeysListVersionsResource(Resource): +# def get(self, nsname, key): +# "Get all versions of a given key in namespace" +# return current_app.backend.key_list_versions(nsname, key) +# + +@router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_get(nsname: str, key: str): + """Retrieve a given key's value from a given namespace""" + return app.backend.key_get(nsname, key) + + +@router.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_put(nsname: str, key: str, request: Request): + """Create or update the value of a key""" + # check if "reblaze/k/" exists in system/schema-validation + req_json = await request.json() + + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(req_json, schema) + if isValid is False: + raise HTTPException(500, "schema mismatched") + return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + + +@router.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_delete(nsname: str, key: str, request: Request): + """Delete a key""" + return app.backend.key_delete(nsname, key, get_gitactor(request)) + + +# @ns_db.route("//k//") +# class KeyResource(Resource): +# # def get(self, nsname, key): +# # "Retrieve a given key's value from a given namespace" +# # return current_app.backend.key_get(nsname, key) +# +# # def put(self, nsname, key): +# # "Create or update the value of a key" +# # # check if "reblaze/k/" exists in system/schema-validation +# # if nsname != "system": +# # keyName = nsname + "/k/" + key +# # schemas = current_app.backend.key_get("system", "schema-validation") +# # schema = None +# # # find schema if exists and validate the json input +# # for item in schemas.items(): +# # if item[0] == keyName: +# # schema = item[1] +# # break +# # if schema: +# # isValid = validateDbJson(request.json, schema) +# # if isValid is False: +# # abort(500, "schema mismatched") +# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) +# +# # def delete(self, nsname, key): +# # "Delete a key" +# # return current_app.backend.key_delete(nsname, key, get_gitactor()) ############# @@ -929,82 +1619,151 @@ def delete(self, nsname, key): ############# -req_fetch_parser = reqparse.RequestParser() -req_fetch_parser.add_argument("url", location="args", help="url to retrieve") - - -@ns_tools.route("/fetch") -class FetchResource(Resource): - @ns_tools.expect(req_fetch_parser, validate=True) - def get(self): - "Fetch an URL" - args = req_fetch_parser.parse_args() +@router.get("/tools/fetch", tags=[Tags.tools]) +async def fetch_resource_get(url: str): + """Fetch an URL""" + try: + r = requests.get(url) + except Exception as e: + raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) + return r.content + + +# @ns_tools.route("/fetch") +# class FetchResource(Resource): +# @ns_tools.expect(req_fetch_parser, validate=True) +# def get(self): +# "Fetch an URL" +# args = req_fetch_parser.parse_args() +# try: +# r = requests.get(args.url) +# except Exception as e: +# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) +# return make_response(r.content) + + +@router.put("/tools/publish/{config}/", tags=[Tags.tools]) +@router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) +async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): + """Push configuration to s3 buckets""" + conf = app.backend.configs_get(config, version) + ok = True + status = [] + req_json = await request.json() + if type(req_json) is not list: + raise HTTPException(400, "body must be a list") + for bucket in buckets: + logs = [] try: - r = requests.get(args.url) + cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) except Exception as e: - abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) - return make_response(r.content) - - -@ns_tools.route("/publish//") -@ns_tools.route("/publish//v//") -class PublishResource(Resource): - @ns_tools.expect([m_bucket], validate=True) - def put(self, config, version=None): - "Push configuration to s3 buckets" - conf = current_app.backend.configs_get(config, version) - ok = True - status = [] - if type(request.json) is not list: - abort(400, "body must be a list") - for bucket in request.json: - logs = [] - try: - cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitpush/") -class GitPushResource(Resource): - @ns_tools.expect([m_giturl], validate=True) - def put(self): - "Push git configuration to remote git repositories" - ok = True - status = [] - for giturl in request.json: - try: - current_app.backend.gitpush(giturl["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": giturl["giturl"], "ok": s, "message": msg}) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitfetch/") -class GitFetchResource(Resource): - @ns_tools.expect(m_giturl, validate=True) - def put(self): - "Fetch git configuration from specified remote repository" - ok = True + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + ) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/publish//") +# @ns_tools.route("/publish//v//") +# class PublishResource(Resource): +# @ns_tools.expect([m_bucket], validate=True) +# def put(self, config, version=None): +# "Push configuration to s3 buckets" +# conf = current_app.backend.configs_get(config, version) +# ok = True +# status = [] +# if type(request.json) is not list: +# abort(400, "body must be a list") +# for bucket in request.json: +# logs = [] +# try: +# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) +# except Exception as e: +# ok = False +# s = False +# msg = repr(e) +# else: +# s = True +# msg = "ok" +# status.append( +# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} +# ) +# return make_response({"ok": ok, "status": status}) + + +@router.put("/tools/gitpush/", tags=[Tags.tools]) +async def git_push_resource_put(git_urls: List[GitUrl]): + """Push git configuration to remote git repositories""" + ok = True + status = [] + for giturl in git_urls: try: - current_app.backend.gitfetch(request.json["giturl"]) + app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: - ok = False msg = repr(e) + s = False else: msg = "ok" - return make_response({"ok": ok, "status": msg}) + s = True + status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/gitpush/") +# class GitPushResource(Resource): +# @ns_tools.expect([m_giturl], validate=True) +# def put(self): +# "Push git configuration to remote git repositories" +# ok = True +# status = [] +# for giturl in request.json: +# try: +# current_app.backend.gitpush(giturl["giturl"]) +# except Exception as e: +# msg = repr(e) +# s = False +# else: +# msg = "ok" +# s = True +# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) +# return make_response({"ok": ok, "status": status}) + + +@router.put("/tools/gitfetch/", tags=[Tags.tools]) +async def git_fetch_resource_put(giturl: GitUrl): + """Fetch git configuration from specified remote repository""" + ok = True + try: + app.backend.gitfetch(dict(giturl)["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return {"ok": ok, "status": msg} + + +# @ns_tools.route("/gitfetch/") +# class GitFetchResource(Resource): +# @ns_tools.expect(m_giturl, validate=True) +# def put(self): +# "Fetch git configuration from specified remote repository" +# ok = True +# try: +# current_app.backend.gitfetch(request.json["giturl"]) +# except Exception as e: +# ok = False +# msg = repr(e) +# else: +# msg = "ok" +# return make_response({"ok": ok, "status": msg}) + + +if __name__ == '__main__': + print("hi") diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index 1f5b8dbe5..8bcd608ba 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -38,6 +38,7 @@ "fasteners", "jsonpath-ng==1.5.3", "pydash==5.0.2", + "fastapi==0.87.0" ], classifiers=[ "Programming Language :: Python :: 3", From 80016c1979969f5f3bcd11f4dc104430a01841a6 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 14:54:14 +0200 Subject: [PATCH 08/55] adding request variable --- .../server/curieconf/confserver/v3/api.py | 195 +++++++++--------- 1 file changed, 97 insertions(+), 98 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 8cc9c0781..dd9ddd7bf 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -14,7 +14,6 @@ # from curieconf import utils from curieconf.utils import cloud -# from curieconf.confserver import app # TODO: TEMP DEFINITIONS @@ -107,14 +106,14 @@ class Limit(BaseModel): # securitypolicy class SecProfileMap(BaseModel): - id: str - name: str - description: str - match: str - acl_profile: str - acl_active: bool - content_filter_profile: str - content_filter_active: bool + id: str = None + name: str = None + description: Optional[str] + match: str = None + acl_profile: str = None + acl_active: bool = None + content_filter_profile: str = None + content_filter_active: bool = None limit_ids: Optional[list] @@ -141,12 +140,12 @@ class SecProfileMap(BaseModel): class SecurityPolicy(BaseModel): id: str name: str - description: str - tags: List[str] + description: Optional[str] + tags: Optional[List[str]] match: str session: anyTypeUnion session_ids: anyTypeUnion - map: List[SecProfileMap] + map: Optional[List[SecProfileMap]] # m_securitypolicy = api.model( @@ -566,10 +565,10 @@ class ConfigDocuments(BaseModel): class ConfigBlobs(BaseModel): - geolite2asn: Optional[List[Optional[BlobEntry]]] - geolite2country: Optional[List[Optional[BlobEntry]]] - geolite2city: Optional[List[Optional[BlobEntry]]] - customconf: Optional[List[Optional[BlobEntry]]] + geolite2asn: Optional[BlobEntry] + geolite2country: Optional[BlobEntry] + geolite2city: Optional[BlobEntry] + customconf: Optional[BlobEntry] # m_config_blobs = api.model( @@ -700,13 +699,13 @@ def validateDbJson(json_data, schema): def get_gitactor(request): email, username = "", "" - email_header = app.options.get("trusted_email_header", None) + email_header = request.app.options.get("trusted_email_header", None) if email_header: email = request.headers.get(email_header, "") - username_header = app.options.get("trusted_username_header", None) + username_header = request.app.options.get("trusted_username_header", None) if username_header: username = request.headers.get(username_header, "") - return app.backend.prepare_actor(username, email) + return request.app.backend.prepare_actor(username, email) base_path = Path(__file__).parent @@ -779,8 +778,8 @@ async def configs_get(request: Request): @router.post("/configs/", tags=[Tags.congifs]) async def configs_post(config: Config, request: Request): """Create a new configuration""" - data = dict(config) - return request.app.backend.configs_create(data, get_gitactor(request)) + data = await request.json() + return request.app.backend.configs_create(data=data, actor = get_gitactor(request)) # @@ -798,29 +797,29 @@ async def configs_post(config: Config, request: Request): # return current_app.backend.configs_create(data, get_gitactor()) @router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) -async def config_get(config: str): +async def config_get(config: str, request: Request): """Retrieve a complete configuration""" - return app.backend.configs_get(config) + return request.app.backend.configs_get(config) @router.post("/configs/{config}/", tags=[Tags.congifs]) async def config_post(config: str, m_config: Config, request: Request): "Create a new configuration. Configuration name in URL overrides configuration in POST data" data = dict(m_config) - return app.backend.configs_create(data, config, get_gitactor(request)) + return request.app.backend.configs_create(data, config, get_gitactor(request)) @router.put("/configs/{config}/", tags=[Tags.congifs]) async def config_put(config: str, meta: Meta, request: Request): """Update an existing configuration""" data = dict(meta) - return app.backend.configs_update(config, data, get_gitactor(request)) + return request.app.backend.configs_update(config, data, get_gitactor(request)) @router.delete("/configs/{config}/", tags=[Tags.congifs]) -async def config_delete(config: str): +async def config_delete(config: str, request: Request): """Delete a configuration""" - return app.backend.configs_delete(config) + return request.app.backend.configs_delete(config) # @ns_configs.route("//") @@ -848,10 +847,10 @@ async def config_delete(config: str): @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) -async def config_clone_post(config: str, meta: Meta): +async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" data = dict(meta) - return app.backend.configs_clone(config, data) + return request.app.backend.configs_clone(config, data) # @ns_configs.route("//clone/") @@ -864,10 +863,10 @@ async def config_clone_post(config: str, meta: Meta): # @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta): +async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): """Clone a configuration. New name is provided URL""" data = dict(meta) - return app.backend.configs_clone(config, data, new_name) + return request.app.backend.configs_clone(config, data, new_name) # @ns_configs.route("//clone//") @@ -880,9 +879,9 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta): @router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def config_list_version_get(config: str): +async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" - return app.backend.configs_list_versions(config) + return request.app.backend.configs_list_versions(config) # @ns_configs.route("//v/") @@ -894,9 +893,9 @@ async def config_list_version_get(config: str): @router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) -async def config_version_get(config: str, version: str): +async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" - return app.backend.configs_get(config, version) + return request.app.backend.configs_get(config, version) # @ns_configs.route("//v//") @@ -908,7 +907,7 @@ async def config_version_get(config: str, version: str): @router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" - return app.backend.configs_revert(config, version, get_gitactor(request)) + return request.app.backend.configs_revert(config, version, get_gitactor(request)) # @ns_configs.route("//v//revert/") @@ -924,9 +923,9 @@ async def config_revert_put(config: str, version: str, request: Request): @router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) -async def blobs_resource_get(config: str): +async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" - res = app.backend.blobs_list(config) + res = request.app.backend.blobs_list(config) return res @@ -939,15 +938,15 @@ async def blobs_resource_get(config: str): # return res @router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_resource_get(config: str, blob: str): +async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" - return app.backend.blobs_get(config, blob) + return request.app.backend.blobs_get(config, blob) @router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" - return app.backend.blobs_create( + return request.app.backend.blobs_create( config, blob, dict(blob_entry), get_gitactor(request) ) @@ -955,7 +954,7 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ @router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" - return app.backend.blobs_update( + return request.app.backend.blobs_update( config, blob, dict(blob_entry), get_gitactor(request) ) @@ -963,7 +962,7 @@ async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, reque @router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_get(config: str, blob: str, request: Request): """Delete a blob""" - return app.backend.blobs_delete(config, blob, get_gitactor(request)) + return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) # @@ -994,9 +993,9 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def blob_list_version_resource_get(config: str, blob: str): +async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" - res = app.backend.blobs_list_versions(config, blob) + res = request.app.backend.blobs_list_versions(config, blob) return res @@ -1010,9 +1009,9 @@ async def blob_list_version_resource_get(config: str, blob: str): @router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) -async def blob_version_resource_get(config: str, blob: str, version: str): +async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" - return app.backend.blobs_get(config, blob, version) + return request.app.backend.blobs_get(config, blob, version) # @ns_configs.route("//b//v//") @@ -1025,7 +1024,7 @@ async def blob_version_resource_get(config: str, blob: str, version: str): @router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): """Create a new version for a blob from an old version""" - return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) # @@ -1041,9 +1040,9 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ################# @router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) -async def document_resource(config: str): +async def document_resource(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" - res = app.backend.documents_list(config) + res = request.app.backend.documents_list(config) return res @@ -1058,11 +1057,11 @@ async def document_resource(config: str): @router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) -async def document_resource_get(config: str, document: str): +async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - res = app.backend.documents_get(config, document) + res = request.app.backend.documents_get(config, document) res = {key: res[key] for key in list(models[document].__fields__.keys())} return res @@ -1082,7 +1081,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) - res = app.backend.documents_create( + res = request.app.backend.documents_create( config, document, data, get_gitactor(request) ) return res @@ -1099,7 +1098,7 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = app.backend.documents_update( + res = request.app.backend.documents_update( config, document, data, get_gitactor(request) ) return res @@ -1110,7 +1109,7 @@ async def document_resource_delete(config: str, document: str, request: Request) """Delete/empty a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_delete(config, document, get_gitactor(request)) + res = request.app.backend.documents_delete(config, document, get_gitactor(request)) return res @@ -1163,11 +1162,11 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str): +async def document_list_version_resource_get(config: str, document: str, request: Request): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_list_versions(config, document) + res = request.app.backend.documents_list_versions(config, document) res = {key: res[key] for key in list(VersionLog.__fields__.keys())} return res @@ -1184,11 +1183,11 @@ async def document_list_version_resource_get(config: str, document: str): @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str): +async def document_version_resource_get(config: str, document: str, version: str, request: Request): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_get(config, document, version) + res = request.app.backend.documents_get(config, document, version) return {key: res[key] for key in list(models[document].__fields__.keys())} @@ -1204,7 +1203,7 @@ async def document_version_resource_get(config: str, document: str, version: str @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) async def document_revert_resource_put(config: str, document: str, version: str, request: Request): """Create a new version for a document from an old version""" - return app.backend.documents_revert( + return request.app.backend.documents_revert( config, document, version, get_gitactor(request) ) @@ -1223,11 +1222,11 @@ async def document_revert_resource_put(config: str, document: str, version: str, ############### @router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_get(config: str, document: str): +async def entries_resource_get(config: str, document: str, request: Request): """Retrieve the list of entries in a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_list(config, document) + res = request.app.backend.entries_list(config, document) return res # XXX: marshal @@ -1239,7 +1238,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn isValid, err = validateJson(dict(basic_entry), document) if isValid: data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_create( + res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) return res @@ -1273,11 +1272,11 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_get(config: str, document: str, entry: str): +async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry) + res = request.app.backend.entries_get(config, document, entry) return {key: res for key in list(models[document].__fields__.keys())} @@ -1290,7 +1289,7 @@ async def entry_resource_put(config: str, document: str, entry: str, basic_entry if isValid: data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_update( + res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) ) return res @@ -1303,7 +1302,7 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: """Delete an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_delete( + res = request.app.backend.entries_delete( config, document, entry, get_gitactor(request) ) return res @@ -1344,11 +1343,11 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: @router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str): +async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_list_versions(config, document, entry) + res = request.app.backend.entries_list_versions(config, document, entry) return {key: res[key] for key in list(VersionLog.__fields__.keys())} @@ -1364,11 +1363,11 @@ async def entry_list_version_resource_get(config: str, document: str, entry: str @router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str): +async def entry_version_resource_get(config: str, document: str, entry: str, version: str, request: Request): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry, version) + res = request.app.backend.entries_get(config, document, entry, version) return {key: res[key] for key in list(models[document].__fields__.keys())} @@ -1389,9 +1388,9 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver ################ @router.get("/db/", tags=[Tags.db]) -async def db_resource_get(): +async def db_resource_get(request: Request): """Get the list of existing namespaces""" - return app.backend.ns_list() + return request.app.backend.ns_list() # @ns_db.route("/") @@ -1402,9 +1401,9 @@ async def db_resource_get(): @router.get("/db/v/", tags=[Tags.db]) -async def db_query_resource_get(): +async def db_query_resource_get(request: Request): """List all existing versions of namespaces""" - return app.backend.ns_list_versions() + return request.app.backend.ns_list_versions() # @@ -1416,10 +1415,10 @@ async def db_query_resource_get(): @router.get("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_get(nsname: str): +async def ns_resource_get(nsname: str, request: Request): """Get a complete namespace""" try: - return app.backend.ns_get(nsname, version=None) + return request.app.backend.ns_get(nsname, version=None) except KeyError: raise HTTPException(404, "namespace [%s] does not exist" % nsname) @@ -1428,7 +1427,7 @@ async def ns_resource_get(nsname: str): async def ns_resource_post(nsname: str, db: DB, request: Request): """Create a non-existing namespace from data""" try: - return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_create(nsname, dict(db), get_gitactor(request)) except Exception: raise HTTPException(409, "namespace [%s] already exists" % nsname) @@ -1436,14 +1435,14 @@ async def ns_resource_post(nsname: str, db: DB, request: Request): @router.put("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, db: DB, request: Request): """Merge data into a namespace""" - return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_update(nsname, dict(db), get_gitactor(request)) @router.delete("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, request: Request): """Delete an existing namespace""" try: - return app.backend.ns_delete(nsname, get_gitactor(request)) + return request.app.backend.ns_delete(nsname, get_gitactor(request)) except KeyError: raise HTTPException(409, "namespace [%s] does not exist" % nsname) @@ -1479,9 +1478,9 @@ async def ns_resource_put(nsname: str, request: Request): @router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) -async def ns_version_resource_get(nsname: str, version: str): +async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" - return app.backend.ns_get(nsname, version) + return request.app.backend.ns_get(nsname, version) # @ns_db.route("//v//") @@ -1495,7 +1494,7 @@ async def ns_version_resource_get(nsname: str, version: str): async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): """Create a new version for a namespace from an old version""" try: - return app.backend.ns_revert(nsname, version, get_gitactor(request)) + return request.app.backend.ns_revert(nsname, version, get_gitactor(request)) except KeyError: raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) @@ -1515,7 +1514,7 @@ async def ns_version_revert_resource_put(nsname: str, version: str, request: Req async def ns_query_resource_post(nsname: str, request: Request): """Run a JSON query on the namespace and returns the results""" req_json = await request.json() - return app.backend.ns_query(nsname, req_json) + return request.app.backend.ns_query(nsname, req_json) # @ns_db.route("//q/") @@ -1527,9 +1526,9 @@ async def ns_query_resource_post(nsname: str, request: Request): @router.get("/db/{nsname}/k/", tags=[Tags.db]) -async def keys_resource_get(nsname: str): +async def keys_resource_get(nsname: str, request: Request): """List all keys of a given namespace""" - return app.backend.key_list(nsname) + return request.app.backend.key_list(nsname) # @ns_db.route("//k/") @@ -1539,9 +1538,9 @@ async def keys_resource_get(nsname: str): # return current_app.backend.key_list(nsname) @router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) -async def keys_list_versions_resource_get(nsname: str, key: str): +async def keys_list_versions_resource_get(nsname: str, key: str, request: Request): """Get all versions of a given key in namespace""" - return app.backend.key_list_versions(nsname, key) + return request.app.backend.key_list_versions(nsname, key) # @ns_db.route("//k//v/") @@ -1552,9 +1551,9 @@ async def keys_list_versions_resource_get(nsname: str, key: str): # @router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_get(nsname: str, key: str): +async def key_resource_get(nsname: str, key: str, request: Request): """Retrieve a given key's value from a given namespace""" - return app.backend.key_get(nsname, key) + return request.app.backend.key_get(nsname, key) @router.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) @@ -1565,7 +1564,7 @@ async def key_resource_put(nsname: str, key: str, request: Request): if nsname != "system": keyName = nsname + "/k/" + key - schemas = app.backend.key_get("system", "schema-validation") + schemas = request.app.backend.key_get("system", "schema-validation") schema = None # find schema if exists and validate the json input for item in schemas.items(): @@ -1576,13 +1575,13 @@ async def key_resource_put(nsname: str, key: str, request: Request): isValid = validateDbJson(req_json, schema) if isValid is False: raise HTTPException(500, "schema mismatched") - return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + return request.app.backend.key_set(nsname, key, req_json, get_gitactor(request)) @router.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) async def key_resource_delete(nsname: str, key: str, request: Request): """Delete a key""" - return app.backend.key_delete(nsname, key, get_gitactor(request)) + return request.app.backend.key_delete(nsname, key, get_gitactor(request)) # @ns_db.route("//k//") @@ -1646,7 +1645,7 @@ async def fetch_resource_get(url: str): @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): """Push configuration to s3 buckets""" - conf = app.backend.configs_get(config, version) + conf = request.app.backend.configs_get(config, version) ok = True status = [] req_json = await request.json() @@ -1698,13 +1697,13 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck @router.put("/tools/gitpush/", tags=[Tags.tools]) -async def git_push_resource_put(git_urls: List[GitUrl]): +async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" ok = True status = [] for giturl in git_urls: try: - app.backend.gitpush(dict(giturl)["giturl"]) + request.app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: msg = repr(e) s = False @@ -1736,11 +1735,11 @@ async def git_push_resource_put(git_urls: List[GitUrl]): @router.put("/tools/gitfetch/", tags=[Tags.tools]) -async def git_fetch_resource_put(giturl: GitUrl): +async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True try: - app.backend.gitfetch(dict(giturl)["giturl"]) + request.app.backend.gitfetch(dict(giturl)["giturl"]) except Exception as e: ok = False msg = repr(e) From d08f5836d20df5ebde9b07b5f6913c1338595f04 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 16:33:37 +0200 Subject: [PATCH 09/55] changing to strict --- .../server/curieconf/confserver/v3/api.py | 238 +++++++++--------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index dd9ddd7bf..0fd6f96f2 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,7 +4,7 @@ from typing import Optional, List, Union from fastapi import FastAPI, Request, HTTPException, APIRouter -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt import random # needed for generating a random number for an API import uvicorn # optional if you run it directly from terminal import jsonschema @@ -59,8 +59,8 @@ # limit class Threshold(BaseModel): - limit: int - action: str + limit: StrictInt + action: StrictStr # m_threshold = api.model( @@ -72,18 +72,18 @@ class Threshold(BaseModel): # ) class Limit(BaseModel): - id: str - name: str - description: Optional[str] - _global: bool = Field(alias="global") - active: bool - timeframe: int + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + _global: StrictBool = Field(alias="global") + active: StrictBool + timeframe: StrictInt thresholds: List[Threshold] include: typing.Any exclude: typing.Any key: anyTypeUnion pairwith: typing.Any - tags: List[str] + tags: List[StrictStr] # m_limit = api.model( @@ -106,14 +106,14 @@ class Limit(BaseModel): # securitypolicy class SecProfileMap(BaseModel): - id: str = None - name: str = None - description: Optional[str] - match: str = None - acl_profile: str = None - acl_active: bool = None - content_filter_profile: str = None - content_filter_active: bool = None + id: StrictStr = None + name: StrictStr = None + description: Optional[StrictStr] + match: StrictStr = None + acl_profile: StrictStr = None + acl_active: StrictBool = None + content_filter_profile: StrictStr = None + content_filter_active: StrictBool = None limit_ids: Optional[list] @@ -138,11 +138,11 @@ class SecProfileMap(BaseModel): # ) class SecurityPolicy(BaseModel): - id: str - name: str - description: Optional[str] - tags: Optional[List[str]] - match: str + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + tags: Optional[List[StrictStr]] + match: StrictStr session: anyTypeUnion session_ids: anyTypeUnion map: Optional[List[SecProfileMap]] @@ -165,17 +165,17 @@ class SecurityPolicy(BaseModel): # content filter rule class ContentFilterRule(BaseModel): - id: str - name: str - msg: str - operand: str - severity: int - certainity: int - category: str - subcategory: str - risk: int - tags: Optional[List[str]] - description: Optional[str] + id: StrictStr + name: StrictStr + msg: StrictStr + operand: StrictStr + severity: StrictInt + certainity: StrictInt + category: StrictStr + subcategory: StrictStr + risk: StrictInt + tags: Optional[List[StrictStr]] + description: Optional[StrictStr] # m_contentfilterrule = api.model( @@ -197,24 +197,24 @@ class ContentFilterRule(BaseModel): # content filter profile class ContentFilterProfile(BaseModel): - id: str - name: str - description: Optional[str] - ignore_alphanum: bool + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + ignore_alphanum: StrictBool args: typing.Any headers: typing.Any cookies: typing.Any path: typing.Any allsections: typing.Any decoding: typing.Any - masking_seed: str - content_type: Optional[List[str]] - active: Optional[List[str]] - report: Optional[List[str]] - ignore: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - ignore_body: bool + masking_seed: StrictStr + content_type: Optional[List[StrictStr]] + active: Optional[List[StrictStr]] + report: Optional[List[StrictStr]] + ignore: Optional[List[StrictStr]] + tags: Optional[List[StrictStr]] + action: Optional[StrictStr] + ignore_body: StrictBool # m_contentfilterprofile = api.model( @@ -243,17 +243,17 @@ class ContentFilterProfile(BaseModel): # aclprofile class ACLProfile(BaseModel): - id: str - name: str - description: Optional[str] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + allow: Optional[List[StrictStr]] + allow_bot: Optional[List[StrictStr]] + deny_bot: Optional[List[StrictStr]] + passthrough: Optional[List[StrictStr]] + deny: Optional[List[StrictStr]] + force_deny: Optional[List[StrictStr]] + tags: Optional[List[StrictStr]] + action: Optional[StrictStr] # m_aclprofile = api.model( @@ -275,14 +275,14 @@ class ACLProfile(BaseModel): # Global Filter class GlobalFilter(BaseModel): - id: str - name: str - source: str - mdate: str - description: str - active: bool + id: StrictStr + name: StrictStr + source: StrictStr + mdate: StrictStr + description: StrictStr + active: StrictBool action: typing.Any - tags: Optional[List[str]] + tags: Optional[List[StrictStr]] rule: anyTypeUnion @@ -304,16 +304,16 @@ class GlobalFilter(BaseModel): # Flow Control class FlowControl(BaseModel): - id: str - name: str - timeframe: int + id: StrictStr + name: StrictStr + timeframe: StrictInt key: List[typing.Any] sequence: List[typing.Any] - tags: Optional[List[str]] - include: Optional[List[str]] - exclude: Optional[List[str]] - description: Optional[str] - active: bool + tags: Optional[List[StrictStr]] + include: Optional[List[StrictStr]] + exclude: Optional[List[StrictStr]] + description: Optional[StrictStr] + active: StrictBool # @@ -336,12 +336,12 @@ class FlowControl(BaseModel): # Action class Action(BaseModel): - id: str - name: str - description: Optional[str] - tags: List[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + tags: List[StrictStr] params: typing.Any - type: str + type: StrictStr # m_action = api.model( @@ -358,9 +358,9 @@ class Action(BaseModel): # Virtual Tag class VirtualTag(BaseModel): - id: str - name: str - description: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] match: List[typing.Any] @@ -377,8 +377,8 @@ class VirtualTag(BaseModel): # custom class Custom(BaseModel): - id: str - name: str + id: StrictStr + name: StrictStr # m_custom = api.model( @@ -408,33 +408,33 @@ class Custom(BaseModel): ### Other models class DocumentMask(BaseModel): - id: str - name: str - description: str + id: StrictStr + name: StrictStr + description: StrictStr map: Optional[List[SecProfileMap]] include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] - tags: Optional[List[str]] + tags: Optional[List[StrictStr]] active: Optional[List[typing.Any]] action: typing.Any sequence: Optional[List[typing.Any]] - timeframe: Optional[int] + timeframe: Optional[StrictInt] thresholds: Optional[List[Threshold]] pairwith: typing.Any - content_type: Optional[List[str]] + content_type: Optional[List[StrictStr]] params: typing.Any decoding: typing.Any - category: Optional[str] - subcategory: Optional[str] - risk: Optional[int] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - match: Optional[str] = "j" - _type: Optional[str] = Field(alias="type") + category: Optional[StrictStr] + subcategory: Optional[StrictStr] + risk: Optional[StrictInt] + allow: Optional[List[StrictStr]] + allow_bot: Optional[List[StrictStr]] + deny_bot: Optional[List[StrictStr]] + passthrough: Optional[List[StrictStr]] + deny: Optional[List[StrictStr]] + force_deny: Optional[List[StrictStr]] + match: Optional[StrictStr] = "j" + _type: Optional[StrictStr] = Field(alias="type") _star: Optional[List[typing.Any]] = Field(alias="*") @@ -473,7 +473,7 @@ class DocumentMask(BaseModel): # ) class VersionLog(BaseModel): - version: Optional[str] + version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] _star: Optional[List[typing.Any]] = Field(alias="*") @@ -490,11 +490,11 @@ class VersionLog(BaseModel): # ) class Meta(BaseModel): - id: str - description: str + id: StrictStr + description: StrictStr date: Optional[datetime.datetime] logs: Optional[List[VersionLog]] = [] - version: Optional[str] + version: Optional[StrictStr] # m_meta = api.model( @@ -509,7 +509,7 @@ class Meta(BaseModel): # ) class BlobEntry(BaseModel): - format: str + format: StrictStr blob: anyTypeUnion @@ -522,7 +522,7 @@ class BlobEntry(BaseModel): # ) class BlobListEntry(BaseModel): - name: Optional[str] + name: Optional[StrictStr] # m_blob_list_entry = api.model( @@ -533,8 +533,8 @@ class BlobListEntry(BaseModel): # ) class DocumentListEntry(BaseModel): - name: Optional[str] - entries: Optional[int] + name: Optional[StrictStr] + entries: Optional[StrictInt] # m_document_list_entry = api.model( @@ -577,10 +577,10 @@ class ConfigBlobs(BaseModel): # ) class ConfigDeleteBlobs(BaseModel): - geolite2asn: Optional[bool] - geolite2country: Optional[bool] - geolite2city: Optional[bool] - customconf: Optional[bool] + geolite2asn: Optional[StrictBool] + geolite2country: Optional[StrictBool] + geolite2city: Optional[StrictBool] + customconf: Optional[StrictBool] # m_config_delete_blobs = api.model( @@ -607,8 +607,8 @@ class Config(BaseModel): # ) class Edit(BaseModel): - path: str - value: str + path: StrictStr + value: StrictStr # m_edit = api.model( @@ -620,9 +620,9 @@ class Edit(BaseModel): # ) class BasicEntry(BaseModel): - id: str - name: str - description: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] # m_basic_entry = api.model( @@ -637,8 +637,8 @@ class BasicEntry(BaseModel): ### Publish class Bucket(BaseModel): - name: str - url: str + name: StrictStr + url: StrictStr # m_bucket = api.model( @@ -652,7 +652,7 @@ class Bucket(BaseModel): ### Git push & pull class GitUrl(BaseModel): - giturl: str + giturl: StrictStr # m_giturl = api.model( From 125872ccf1b8400d31212da2f54e4be83a9f820f Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 16:55:54 +0200 Subject: [PATCH 10/55] dict to await json --- .../server/curieconf/confserver/v3/api.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 0fd6f96f2..39778bf78 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -15,7 +15,6 @@ # from curieconf import utils from curieconf.utils import cloud - # TODO: TEMP DEFINITIONS import os @@ -779,7 +778,7 @@ async def configs_get(request: Request): async def configs_post(config: Config, request: Request): """Create a new configuration""" data = await request.json() - return request.app.backend.configs_create(data=data, actor = get_gitactor(request)) + return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) # @@ -805,14 +804,14 @@ async def config_get(config: str, request: Request): @router.post("/configs/{config}/", tags=[Tags.congifs]) async def config_post(config: str, m_config: Config, request: Request): "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = dict(m_config) + data = await request.json() return request.app.backend.configs_create(data, config, get_gitactor(request)) @router.put("/configs/{config}/", tags=[Tags.congifs]) async def config_put(config: str, meta: Meta, request: Request): """Update an existing configuration""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_update(config, data, get_gitactor(request)) @@ -849,7 +848,7 @@ async def config_delete(config: str, request: Request): @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_clone(config, data) @@ -865,7 +864,7 @@ async def config_clone_post(config: str, meta: Meta, request: Request): @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): """Clone a configuration. New name is provided URL""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_clone(config, data, new_name) @@ -946,16 +945,19 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" + b_entry = await request.json() return request.app.backend.blobs_create( - config, blob, dict(blob_entry), get_gitactor(request) + config, blob, b_entry, get_gitactor(request) ) @router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" + """upaate an existing blob""" + b_entry = await request.json() + return request.app.backend.blobs_update( - config, blob, dict(blob_entry), get_gitactor(request) + config, blob, b_entry, get_gitactor(request) ) @@ -1426,8 +1428,9 @@ async def ns_resource_get(nsname: str, request: Request): @router.post("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_post(nsname: str, db: DB, request: Request): """Create a non-existing namespace from data""" + _db = await request.json() try: - return request.app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_create(nsname, _db, get_gitactor(request)) except Exception: raise HTTPException(409, "namespace [%s] already exists" % nsname) @@ -1435,7 +1438,9 @@ async def ns_resource_post(nsname: str, db: DB, request: Request): @router.put("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, db: DB, request: Request): """Merge data into a namespace""" - return request.app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + _db = await request.json() + + return request.app.backend.ns_update(nsname, _db, get_gitactor(request)) @router.delete("/db/{nsname}/", tags=[Tags.db]) @@ -1651,10 +1656,12 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck req_json = await request.json() if type(req_json) is not list: raise HTTPException(400, "body must be a list") + buck = await request.json() + for bucket in buckets: logs = [] try: - cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) + cloud.export(conf, buck["url"], prnt=lambda x: logs.append(x)) except Exception as e: ok = False s = False @@ -1663,7 +1670,7 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck s = True msg = "ok" status.append( - {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + {"name": buck["name"], "ok": s, "logs": logs, "message": msg} ) return {"ok": ok, "status": status} @@ -1738,8 +1745,9 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True + _giturl = await request.json() try: - request.app.backend.gitfetch(dict(giturl)["giturl"]) + request.app.backend.gitfetch(_giturl["giturl"]) except Exception as e: ok = False msg = repr(e) From a5f4b5d13080b01312add61338221b9fc72012c4 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 18:34:46 +0200 Subject: [PATCH 11/55] configs actions of configs --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 39778bf78..7cdf598ae 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -877,7 +877,7 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta, request # return current_app.backend.configs_clone(config, data, new_name) -@router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" return request.app.backend.configs_list_versions(config) @@ -891,7 +891,7 @@ async def config_list_version_get(config: str, request: Request): # return current_app.backend.configs_list_versions(config) -@router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +@router.get("/configs/{config}/v/{version}/", tags=[Tags.congifs]) async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" return request.app.backend.configs_get(config, version) @@ -903,7 +903,7 @@ async def config_version_get(config: str, version: str, request: Request): # "Retrieve a specific version of a configuration" # return current_app.backend.configs_get(config, version) -@router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +@router.get("/configs/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" return request.app.backend.configs_revert(config, version, get_gitactor(request)) From 5d4aa8a4d56276d0b894f83becb959ae75d1ec64 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 09:40:09 +0200 Subject: [PATCH 12/55] pre entries --- .../server/curieconf/confserver/v3/api.py | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 7cdf598ae..be7ed13c4 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -340,8 +340,12 @@ class Action(BaseModel): description: Optional[StrictStr] tags: List[StrictStr] params: typing.Any - type: StrictStr + _type: StrictStr = Field(alias="type") + class Config: + fields = { + "_type": "type" + } # m_action = api.model( # "Action", @@ -409,7 +413,7 @@ class Custom(BaseModel): class DocumentMask(BaseModel): id: StrictStr name: StrictStr - description: StrictStr + description: Optional[StrictStr] map: Optional[List[SecProfileMap]] include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] @@ -432,10 +436,14 @@ class DocumentMask(BaseModel): passthrough: Optional[List[StrictStr]] deny: Optional[List[StrictStr]] force_deny: Optional[List[StrictStr]] - match: Optional[StrictStr] = "j" + match: Optional[StrictStr] _type: Optional[StrictStr] = Field(alias="type") _star: Optional[List[typing.Any]] = Field(alias="*") + class Config: + fields = { + "_type": "type" + } # m_document_mask = api.model( # "Mask for document", @@ -921,7 +929,7 @@ async def config_revert_put(config: str, version: str, request: Request): ############# -@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry]) async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" res = request.app.backend.blobs_list(config) @@ -936,13 +944,13 @@ async def blobs_resource_get(config: str, request: Request): # res = current_app.backend.blobs_list(config) # return res -@router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +@router.get("/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" return request.app.backend.blobs_get(config, blob) -@router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +@router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" b_entry = await request.json() @@ -951,7 +959,7 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ ) -@router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +@router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): """upaate an existing blob""" b_entry = await request.json() @@ -961,8 +969,8 @@ async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, reque ) -@router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_get(config: str, blob: str, request: Request): +@router.delete("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_delete(config: str, blob: str, request: Request): """Delete a blob""" return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) @@ -994,7 +1002,7 @@ async def blob_resource_get(config: str, blob: str, request: Request): # return current_app.backend.blobs_delete(config, blob, get_gitactor()) -@router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" res = request.app.backend.blobs_list_versions(config, blob) @@ -1010,7 +1018,7 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques # return res -@router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=VersionLog) async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" return request.app.backend.blobs_get(config, blob, version) @@ -1023,7 +1031,7 @@ async def blob_version_resource_get(config: str, blob: str, version: str, reques # "Retrieve the given version of a blob" # return current_app.backend.blobs_get(config, blob, version) -@router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +@router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): """Create a new version for a blob from an old version""" return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) @@ -1041,7 +1049,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ### DOCUMENTS ### ################# -@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) async def document_resource(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) @@ -1058,28 +1066,37 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True) async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - res = {key: res[key] for key in list(models[document].__fields__.keys())} + #res = {key: res[key] for key in list(models[document].__fields__.keys())} return res async def _filter(data, keys): - return {key: data[key] for key in keys} + filtered = {} + for key in keys: + if data.get(key, False): + filtered[key] = data[key] + return filtered + + @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): """Create a new complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: + if document not in models.keys(): + raise HTTPException(status_code=404, detail="document name is not one of the possible name in 'models' module") + + as_dict = await request.json() + print(as_dict[0]) + data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] + print(data[0]) + for entry in data: isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) @@ -1094,8 +1111,8 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + as_dict = await request.json() + data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] for entry in basic_entries: isValid, err = validateJson(dict(entry), document) if isValid is False: @@ -1164,12 +1181,12 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request): +async def document_list_version_resource_get(config: str, document: str, request: Request, response_model = List[VersionLog]): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_list_versions(config, document) - res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + #res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] return res @@ -1190,7 +1207,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} + return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False) } for r in res] # @ns_configs.route("//d//v//") From b09640b8f880863928f92f0789629469e246f672 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 12:35:48 +0200 Subject: [PATCH 13/55] config done prior to fixes --- .../server/curieconf/confserver/v3/api.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index be7ed13c4..2b8df33a7 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -74,7 +74,7 @@ class Limit(BaseModel): id: StrictStr name: StrictStr description: Optional[StrictStr] - _global: StrictBool = Field(alias="global") + global_: StrictBool = Field(alias="global") active: StrictBool timeframe: StrictInt thresholds: List[Threshold] @@ -340,7 +340,7 @@ class Action(BaseModel): description: Optional[StrictStr] tags: List[StrictStr] params: typing.Any - _type: StrictStr = Field(alias="type") + type_: StrictStr = Field(alias="type") class Config: fields = { @@ -418,7 +418,7 @@ class DocumentMask(BaseModel): include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] tags: Optional[List[StrictStr]] - active: Optional[List[typing.Any]] + active: Optional[typing.Any] action: typing.Any sequence: Optional[List[typing.Any]] timeframe: Optional[StrictInt] @@ -437,8 +437,8 @@ class DocumentMask(BaseModel): deny: Optional[List[StrictStr]] force_deny: Optional[List[StrictStr]] match: Optional[StrictStr] - _type: Optional[StrictStr] = Field(alias="type") - _star: Optional[List[typing.Any]] = Field(alias="*") + type_: Optional[StrictStr] = Field(alias="type") + star_: Optional[List[typing.Any]] = Field(alias="*") class Config: fields = { @@ -483,7 +483,7 @@ class VersionLog(BaseModel): version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] - _star: Optional[List[typing.Any]] = Field(alias="*") + star_: Optional[List[typing.Any]] = Field(alias="*") # @@ -1066,12 +1066,13 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True, response_model_by_alias=True) async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) + print(res[0]["global"]) #res = {key: res[key] for key in list(models[document].__fields__.keys())} return res @@ -1252,11 +1253,13 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): "Create an entry in a document" + + data_json = await request.json() if document not in models: raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) + isValid, err = validateJson(data_json, document) if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + data = {key: data_json[key] for key in list(models[document].__fields__.keys())} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) @@ -1290,23 +1293,28 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn # abort(500, "schema mismatched: \n" + err) +def switch_alias(keys): + return [key[:-1] if key.endswith("_") else key for key in keys] + @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) - return {key: res for key in list(models[document].__fields__.keys())} + keys = switch_alias(list(models[document].__fields__.keys())) + return {key: res[key] for key in keys} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): """Update an entry in a document""" + data_json = await request.json() if document not in models: raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) + isValid, err = validateJson(data_json, document) if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys()))} res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1790,4 +1798,5 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): if __name__ == '__main__': - print("hi") + l = ["k_", "jjjj", "lama_"] + print(switch_alias(l)) \ No newline at end of file From baa07f930c3c0f7c8ec250464682b987ab0dfdb4 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 14:32:50 +0200 Subject: [PATCH 14/55] db done except /q/ --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 2b8df33a7..4a95b4fe3 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1507,7 +1507,7 @@ async def ns_resource_put(nsname: str, request: Request): # # abort(409, "namespace [%s] does not exist" % nsname) -@router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +@router.get("/db/{nsname}/v/{version}/", tags=[Tags.db]) async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" return request.app.backend.ns_get(nsname, version) From b3ec478b0d3d3099ae795280fbd413976baaf09b Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 14:55:32 +0200 Subject: [PATCH 15/55] done without fixes --- .../server/curieconf/confserver/v3/api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 4a95b4fe3..f147ed653 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1678,15 +1678,14 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck conf = request.app.backend.configs_get(config, version) ok = True status = [] - req_json = await request.json() - if type(req_json) is not list: + buckets = await request.json() + if type(buckets) is not list: raise HTTPException(400, "body must be a list") - buck = await request.json() for bucket in buckets: logs = [] try: - cloud.export(conf, buck["url"], prnt=lambda x: logs.append(x)) + cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) except Exception as e: ok = False s = False @@ -1733,16 +1732,17 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" ok = True status = [] - for giturl in git_urls: + git_jsons = await request.json() + for giturl in git_jsons: try: - request.app.backend.gitpush(dict(giturl)["giturl"]) + request.app.backend.gitpush(giturl["giturl"]) except Exception as e: msg = repr(e) s = False else: msg = "ok" s = True - status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + status.append({"url": giturl["giturl"], "ok": s, "message": msg}) return {"ok": ok, "status": status} @@ -1770,9 +1770,9 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True - _giturl = await request.json() + giturl_json = await request.json() try: - request.app.backend.gitfetch(_giturl["giturl"]) + request.app.backend.gitfetch(giturl_json["giturl"]) except Exception as e: ok = False msg = repr(e) From a0f26f8701646b6a6c6ce070d98844128f56f9ac Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 19:41:52 +0200 Subject: [PATCH 16/55] Version Log x-fields --- .../server/curieconf/confserver/v3/api.py | 99 +++++++++++++------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index f147ed653..0619cf082 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -18,7 +18,7 @@ # TODO: TEMP DEFINITIONS import os -router = APIRouter() +router = APIRouter(prefix="/api/v3") options = {} val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) if val: @@ -347,6 +347,7 @@ class Config: "_type": "type" } + # m_action = api.model( # "Action", # { @@ -445,6 +446,7 @@ class Config: "_type": "type" } + # m_document_mask = api.model( # "Mask for document", # { @@ -875,20 +877,35 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta, request data = await request.json() return request.app.backend.configs_clone(config, data, new_name) - -# @ns_configs.route("//clone//") -# class ConfigCloneName(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config, new_name): -# "Clone a configuration. New name is provided URL" -# data = request.json -# return current_app.backend.configs_clone(config, data, new_name) + # @ns_configs.route("//clone//") + # class ConfigCloneName(Resource): + # @ns_configs.expect(m_meta, validate=True) + # def post(self, config, new_name): + # "Clone a configuration. New name is provided URL" + # data = request.json + # return current_app.backend.configs_clone(config, data, new_name) + + +def filter_x_fields(res, x_fields): + fields = [] + if x_fields.startswith(('[', '{', '(')): + x_fields = x_fields[1:-1] + x_fields = x_fields.replace(" ", "") + fields = x_fields.split(",") + if isinstance(res, list): + return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + else: + return {field: res[field] for field in fields} -@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" - return request.app.backend.configs_list_versions(config) + res = request.app.backend.configs_list_versions(config) + if request.headers.get("X-fields", False): + res = filter_x_fields(res, request.headers["X-fields"]) + return res # @ns_configs.route("//v/") @@ -1002,10 +1019,13 @@ async def blob_resource_delete(config: str, blob: str, request: Request): # return current_app.backend.blobs_delete(config, blob, get_gitactor()) -@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" res = request.app.backend.blobs_list_versions(config, blob) + if request.headers.get("X-fields", False): + res = filter_x_fields(res, request.headers["X-fields"]) return res @@ -1018,9 +1038,13 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques # return res -@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=BlobEntry) async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" + + res = request.app.backend.blobs_get(config, blob, version) + if request.headers.get("X-fields", False): + res = filter_x_fields([res], request.headers["X-fields"]) return request.app.backend.blobs_get(config, blob, version) @@ -1050,7 +1074,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ################# @router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) -async def document_resource(config: str, request: Request): +async def document_resource_get(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) return res @@ -1066,15 +1090,32 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True, response_model_by_alias=True) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_get(config: str, document: str, request: Request): + def filter_document_mask(res, x_fields): + fields = [] + if x_fields: + if x_fields.startswith(('[', '{', '(')): + x_fields = x_fields[1:-1] + x_fields = x_fields.replace(" ", "") + fields = x_fields.split(",") + else: + fields = switch_alias(DocumentMask.__fields__.keys()) + + print(f"fields are {fields}") + print(f"res is {res}") + + return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + """Get a complete document""" + + headers = request.headers + print(headers) if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - print(res[0]["global"]) - #res = {key: res[key] for key in list(models[document].__fields__.keys())} - return res + # res = {key: res[key] for key in list(models[document].__fields__.keys())} + return filter_document_mask(res, headers.get("x-fields", None)) async def _filter(data, keys): @@ -1085,8 +1126,6 @@ async def _filter(data, keys): return filtered - - @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): """Create a new complete document""" @@ -1098,7 +1137,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] print(data[0]) for entry in data: - isValid, err = validateJson(dict(entry), document) + isValid, err = validateJson(data, document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) res = request.app.backend.documents_create( @@ -1113,9 +1152,9 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) + data = [await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) for entry in as_dict] + for entry in data: + isValid, err = validateJson(entry, document) if isValid is False: raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) res = request.app.backend.documents_update( @@ -1182,12 +1221,13 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request, response_model = List[VersionLog]): +async def document_list_version_resource_get(config: str, document: str, request: Request, + response_model=List[VersionLog]): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_list_versions(config, document) - #res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] + # res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] return res @@ -1208,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False) } for r in res] + return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False)} for r in res] # @ns_configs.route("//d//v//") @@ -1296,6 +1336,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn def switch_alias(keys): return [key[:-1] if key.endswith("_") else key for key in keys] + @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" @@ -1303,7 +1344,7 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) keys = switch_alias(list(models[document].__fields__.keys())) - return {key: res[key] for key in keys} + return {key: res[key] for key in keys if res.get(key, False)} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) @@ -1799,4 +1840,4 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): if __name__ == '__main__': l = ["k_", "jjjj", "lama_"] - print(switch_alias(l)) \ No newline at end of file + print(switch_alias(l)) From d104f45179fe62d72c56728351b4835804178e65 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Sun, 27 Nov 2022 14:40:26 +0200 Subject: [PATCH 17/55] cleanup --- .../server/curieconf/confserver/v3/api.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 0619cf082..e607bc168 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1022,7 +1022,7 @@ async def blob_resource_delete(config: str, blob: str, request: Request): @router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], response_model_exclude_unset=True) async def blob_list_version_resource_get(config: str, blob: str, request: Request): - "Retrieve the list of versions of a given blob" + """Retrieve the list of versions of a given blob""" res = request.app.backend.blobs_list_versions(config, blob) if request.headers.get("X-fields", False): res = filter_x_fields(res, request.headers["X-fields"]) @@ -1410,13 +1410,16 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: # # return res -@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request): +@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) +async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request, + ): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_list_versions(config, document, entry) - return {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res + # return {key: res[key] for key in list(VersionLog.__fields__.keys())} # @@ -1836,8 +1839,3 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): # else: # msg = "ok" # return make_response({"ok": ok, "status": msg}) - - -if __name__ == '__main__': - l = ["k_", "jjjj", "lama_"] - print(switch_alias(l)) From 05a6f7745ec0cc5b4085a27fcb1ade12a6d2153b Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Wed, 30 Nov 2022 10:54:04 +0200 Subject: [PATCH 18/55] before fork --- .../server/curieconf/confserver/__init__.py | 47 +++++++++++++++++-- .../confserver/backend/gitbackend.py | 2 +- .../server/curieconf/confserver/v3/api.py | 39 ++++++++------- curiefense/curieconf/server/setup.py | 5 +- curiefense/images/build-docker-images.sh | 2 +- curiefense/images/confserver/Dockerfile | 2 +- .../confserver/bootstrap/bootstrap_config.sh | 2 +- .../images/confserver/init/requirements.txt | 9 ++-- 8 files changed, 75 insertions(+), 33 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 0dbe9486f..870477c73 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -1,17 +1,50 @@ #! /usr/bin/env python3 - +import json import os + from .backend import Backends import uvicorn +import logging from curieconf.confserver.v3 import api -from prometheus_flask_exporter import PrometheusMetrics -from fastapi import FastAPI +from fastapi import FastAPI, Request, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.encoders import jsonable_encoder +from prometheus_fastapi_instrumentator import Instrumentator -app = FastAPI() +app = FastAPI(docs_url="/api/v3/") app.include_router(api.router) +@app.on_event("startup") +async def startup(): + Instrumentator().instrument(app).expose(app) + + +logging.basicConfig( + handlers=[ + logging.FileHandler("fastapi.log"), + logging.StreamHandler() + ], + level=logging.INFO, + format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', + datefmt='%H:%M:%S' +) +logger = logging.getLogger("filters-maxmind") + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + # exc_str = f'{exc}'.replace('\n', ' ').replace(' ', ' ') + # # or logger.error(f'{exc}') + # logger.error(exc_str) + # content = {'status_code': 10422, 'message': exc_str, 'data': None} + # + # return JSONResponse + return PlainTextResponse(str(exc), status_code=400) + + def drop_into_pdb(app, exception): import sys import pdb @@ -49,7 +82,7 @@ def main(args=None): options = parser.parse_args(args) - #TODO - find replacements for got_request_exception and prometheus_flask_exporter + # TODO - find replacements for got_request_exception and prometheus_flask_exporter # if options.pdb: # flask.got_request_exception.connect(drop_into_pdb) # metrics = PrometheusMetrics(app) @@ -62,3 +95,7 @@ def main(args=None): # app.run(debug=options.debug, host=options.host, port=options.port) finally: pass + + +if __name__ == '__main__': + main() diff --git a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py index d22e5a6d3..d81e1a22c 100644 --- a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py +++ b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py @@ -377,7 +377,7 @@ def configs_update(self, config, data, actor=CURIE_AUTHOR): doc = self.update_doc(doc, addd[docname]) if docname in deld: deleid = { - eid for eid, val in deld[docname].items() if val is True + entry["id"] for entry in deld[docname] } doc = [entry for entry in doc if entry["id"] not in deleid] self.add_document(docname, doc) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index e607bc168..a56925d41 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -777,7 +777,7 @@ class Tags(Enum): ### CONFIGS ### ################ -@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta]) +@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta], response_model_exclude_unset=True) async def configs_get(request: Request): """Get the detailed list of existing configurations""" res = request.app.backend.configs_list() @@ -893,9 +893,9 @@ def filter_x_fields(res, x_fields): x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") if isinstance(res, list): - return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + return [{field: r[field] for field in fields if field in r} for r in res] else: - return {field: res[field] for field in fields} + return {field: res[field] for field in fields if field in res} @router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], @@ -1105,7 +1105,7 @@ def filter_document_mask(res, x_fields): print(f"fields are {fields}") print(f"res is {res}") - return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" @@ -1119,11 +1119,12 @@ def filter_document_mask(res, x_fields): async def _filter(data, keys): - filtered = {} - for key in keys: - if data.get(key, False): - filtered[key] = data[key] - return filtered + # filtered = {} + # for key in keys: + # if data.get(key, False): + # filtered[key] = data[key] + # return filtered + return {key: data[key] for key in keys if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) @@ -1220,9 +1221,8 @@ async def document_resource_delete(config: str, document: str, request: Request) # # return res -@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request, - response_model=List[VersionLog]): +@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +async def document_list_version_resource_get(config: str, document: str, request: Request): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1248,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False)} for r in res] + return [{key: r[key] for key in list(models[document].__fields__.keys()) if key in r} for r in res] # @ns_configs.route("//d//v//") @@ -1299,7 +1299,8 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in list(models[document].__fields__.keys())} + keys = list(models[document].__fields__.keys()) + data = {key: data_json[key] for key in keys if key in data_json} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) @@ -1344,18 +1345,19 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) keys = switch_alias(list(models[document].__fields__.keys())) - return {key: res[key] for key in keys if res.get(key, False)} + return {key: res[key] for key in keys if key in res} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): """Update an entry in a document""" data_json = await request.json() + print(f"-------------------------- active: {data_json.get('active', 'blaaaaaaa')} ------------------------------") if document not in models: raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys()))} + data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in data_json} res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1439,7 +1441,8 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} + keys = list(models[document].__fields__.keys()) + return {key: res[key] for key in keys if key in res} # @ns_configs.route( @@ -1738,7 +1741,7 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck s = True msg = "ok" status.append( - {"name": buck["name"], "ok": s, "logs": logs, "message": msg} + {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} ) return {"ok": ok, "status": status} diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index 8bcd608ba..c5144a34f 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -38,7 +38,10 @@ "fasteners", "jsonpath-ng==1.5.3", "pydash==5.0.2", - "fastapi==0.87.0" + "fastapi==0.87.0", + "prometheus-fastapi-instrumentator==5.9.1", + "pydantic==1.10.2", + "uvicorn==0.19.0" ], classifiers=[ "Programming Language :: Python :: 3", diff --git a/curiefense/images/build-docker-images.sh b/curiefense/images/build-docker-images.sh index 6e095204b..bfce9b91a 100755 --- a/curiefense/images/build-docker-images.sh +++ b/curiefense/images/build-docker-images.sh @@ -24,7 +24,7 @@ IFS=' ' read -ra RUST_DISTROS <<< "${RUST_DISTROS:-bionic focal}" if [ -n "$TESTIMG" ]; then IMAGES=("$TESTIMG") OTHER_IMAGES_DOCKER_TAG="$DOCKER_TAG" - DOCKER_TAG="test" + DOCKER_TAG="main" echo "Building only image $TESTIMG" else IMAGES=(confserver curieproxy-istio curieproxy-envoy \ diff --git a/curiefense/images/confserver/Dockerfile b/curiefense/images/confserver/Dockerfile index 8a6310d5b..14988f7b8 100644 --- a/curiefense/images/confserver/Dockerfile +++ b/curiefense/images/confserver/Dockerfile @@ -26,4 +26,4 @@ WORKDIR /app ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"] -CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini", "--callable", "app", "--module", "main"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/curiefense/images/confserver/bootstrap/bootstrap_config.sh b/curiefense/images/confserver/bootstrap/bootstrap_config.sh index 6d0a7a4a5..0574bc30d 100755 --- a/curiefense/images/confserver/bootstrap/bootstrap_config.sh +++ b/curiefense/images/confserver/bootstrap/bootstrap_config.sh @@ -4,7 +4,7 @@ set -e # to be run in an initContainer # Will deploy specified configuration as a bootstrap config, if there is no config in /cf-persistent-config/confdb -TARGETDIR="/cf-persistent-config/confdb" +TARGETDIR="/tmp/cf-persistent-config/confdb" if [ -e "$TARGETDIR" ]; then echo "Config already present in $TARGETDIR, exiting" diff --git a/curiefense/images/confserver/init/requirements.txt b/curiefense/images/confserver/init/requirements.txt index 6a4f4ea71..7ab304acf 100644 --- a/curiefense/images/confserver/init/requirements.txt +++ b/curiefense/images/confserver/init/requirements.txt @@ -15,10 +15,6 @@ configparser==5.2.0 decorator==5.1.1 fasteners==0.17.3 filelock==3.6.0 -Flask==1.1.4 -Flask-Cors==3.0.10 -Flask-PyMongo==2.3.0 -flask-restx==0.5.1 gitdb==4.0.9 GitPython==3.1.27 google-api-core==2.7.2 @@ -61,4 +57,7 @@ uWSGI==2.0.20 Werkzeug==0.16.1 xattr==0.9.9 zipp==3.8.0 -prometheus-flask-exporter==0.20.3 +fastapi==0.87.0 +prometheus-fastapi-instrumentator==5.9.1 +pydantic==1.10.2 +uvicorn==0.19.0 \ No newline at end of file From e2505d7ed9149334811e0e627924a65492656ac2 Mon Sep 17 00:00:00 2001 From: yoavkatzman <100856091+yoavkatzman@users.noreply.github.com> Date: Wed, 30 Nov 2022 17:41:43 +0200 Subject: [PATCH 19/55] Update bootstrap_config.sh --- curiefense/images/confserver/bootstrap/bootstrap_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curiefense/images/confserver/bootstrap/bootstrap_config.sh b/curiefense/images/confserver/bootstrap/bootstrap_config.sh index 0574bc30d..6d0a7a4a5 100755 --- a/curiefense/images/confserver/bootstrap/bootstrap_config.sh +++ b/curiefense/images/confserver/bootstrap/bootstrap_config.sh @@ -4,7 +4,7 @@ set -e # to be run in an initContainer # Will deploy specified configuration as a bootstrap config, if there is no config in /cf-persistent-config/confdb -TARGETDIR="/tmp/cf-persistent-config/confdb" +TARGETDIR="/cf-persistent-config/confdb" if [ -e "$TARGETDIR" ]; then echo "Config already present in $TARGETDIR, exiting" From 863eb54b379864b1f176d9287048e55c5e1cc699 Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 13:20:45 +0200 Subject: [PATCH 20/55] switch alias --- .../curieconf/server/curieconf/confserver/v3/api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index a56925d41..a7af8cd46 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1124,7 +1124,7 @@ async def _filter(data, keys): # if data.get(key, False): # filtered[key] = data[key] # return filtered - return {key: data[key] for key in keys if key in data} + return {key: data[key] for key in switch_alias(keys) if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) @@ -1138,7 +1138,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] print(data[0]) for entry in data: - isValid, err = validateJson(data, document) + isValid, err = validateJson(entry, document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) res = request.app.backend.documents_create( @@ -1248,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if key in r} for r in res] + return [{key: r[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in r} for r in res] # @ns_configs.route("//d//v//") @@ -1299,7 +1299,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - keys = list(models[document].__fields__.keys()) + keys = switch_alias(list(models[document].__fields__.keys())) data = {key: data_json[key] for key in keys if key in data_json} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) @@ -1441,7 +1441,7 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) - keys = list(models[document].__fields__.keys()) + keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @@ -1841,4 +1841,4 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): # msg = repr(e) # else: # msg = "ok" -# return make_response({"ok": ok, "status": msg}) +# return make_response({"ok": ok, "status": msg}) \ No newline at end of file From 08bf59200cca2365a79c8d5589bbaad8beaf387e Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 16:15:14 +0200 Subject: [PATCH 21/55] black fixes --- curiefense/curieconf/server/app/main.py | 1 + .../server/curieconf/confserver/__init__.py | 13 +- .../confserver/backend/gitbackend.py | 4 +- .../server/curieconf/confserver/v3/api.py | 1045 +++-------------- curiefense/curieconf/server/setup.py | 2 +- 5 files changed, 157 insertions(+), 908 deletions(-) diff --git a/curiefense/curieconf/server/app/main.py b/curiefense/curieconf/server/app/main.py index f675059b4..776767a98 100644 --- a/curiefense/curieconf/server/app/main.py +++ b/curiefense/curieconf/server/app/main.py @@ -2,6 +2,7 @@ from curieconf.confserver.backend import Backends import os + app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") options = {} val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index f2b8f20f4..2baa918ac 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -20,18 +20,17 @@ @app.on_event("startup") async def startup(): Instrumentator().instrument(app).expose(app) + + ## Import all versions from .v3 import api as api_v3 logging.basicConfig( - handlers=[ - logging.FileHandler("fastapi.log"), - logging.StreamHandler() - ], + handlers=[logging.FileHandler("fastapi.log"), logging.StreamHandler()], level=logging.INFO, - format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', - datefmt='%H:%M:%S' + format="[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s", + datefmt="%H:%M:%S", ) logger = logging.getLogger("filters-maxmind") @@ -99,5 +98,5 @@ def main(args=None): pass -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py index d81e1a22c..a6c1e3ef0 100644 --- a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py +++ b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py @@ -376,9 +376,7 @@ def configs_update(self, config, data, actor=CURIE_AUTHOR): if docname in addd: doc = self.update_doc(doc, addd[docname]) if docname in deld: - deleid = { - entry["id"] for entry in deld[docname] - } + deleid = {entry["id"] for entry in deld[docname]} doc = [entry for entry in doc if entry["id"] not in deleid] self.add_document(docname, doc) self.commit("Update config [%s]%s" % (config, renamed), actor=actor) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index a7af8cd46..25e9c5a17 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -3,10 +3,8 @@ from enum import Enum from typing import Optional, List, Union -from fastapi import FastAPI, Request, HTTPException, APIRouter +from fastapi import Request, HTTPException, APIRouter from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt -import random # needed for generating a random number for an API -import uvicorn # optional if you run it directly from terminal import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type @@ -32,14 +30,6 @@ from pathlib import Path import json -# api_bp = Blueprint("api_v3", __name__) -# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - -# ns_configs = api.namespace("configs", description="Configurations") -# ns_db = api.namespace("db", description="Database") -# ns_tools = api.namespace("tools", description="Tools") - - ############## ### MODELS ### ############## @@ -51,25 +41,11 @@ anyType = ["number", "string", "boolean", "object", "array", "null"] -# class AnyType(fields.Raw): -# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] - - -# limit - class Threshold(BaseModel): limit: StrictInt action: StrictStr -# m_threshold = api.model( -# "Rate Limit Threshold", -# { -# "limit": fields.Integer(required=True), -# "action": fields.String(required=True), -# }, -# ) - class Limit(BaseModel): id: StrictStr name: StrictStr @@ -85,24 +61,6 @@ class Limit(BaseModel): tags: List[StrictStr] -# m_limit = api.model( -# "Rate Limit", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "global": fields.Boolean(required=True), -# "active": fields.Boolean(required=True), -# "timeframe": fields.Integer(required=True), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "include": fields.Raw(required=True), -# "exclude": fields.Raw(required=True), -# "key": AnyType(required=True), -# "pairwith": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# }, -# ) - # securitypolicy class SecProfileMap(BaseModel): id: StrictStr = None @@ -116,26 +74,6 @@ class SecProfileMap(BaseModel): limit_ids: Optional[list] -# m_secprofilemap = api.model( -# "Security Profile Map", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.String(required=True), -# "acl_profile": fields.String(required=True), -# "acl_active": fields.Boolean(required=True), -# "content_filter_profile": fields.String(required=True), -# "content_filter_active": fields.Boolean(required=True), -# "limit_ids": fields.List(fields.Raw()), -# }, -# ) - -# TODO = deprecated? -# m_map = api.model( -# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -# ) - class SecurityPolicy(BaseModel): id: StrictStr name: StrictStr @@ -147,22 +85,9 @@ class SecurityPolicy(BaseModel): map: Optional[List[SecProfileMap]] -# m_securitypolicy = api.model( -# "Security Policy", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String()), -# "match": fields.String(required=True), -# "session": AnyType(), -# "session_ids": AnyType(), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# }, -# ) - # content filter rule + class ContentFilterRule(BaseModel): id: StrictStr name: StrictStr @@ -177,23 +102,6 @@ class ContentFilterRule(BaseModel): description: Optional[StrictStr] -# m_contentfilterrule = api.model( -# "Content Filter Rule", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "msg": fields.String(required=True), -# "operand": fields.String(required=True), -# "severity": fields.Integer(required=True), -# "certainity": fields.Integer(required=True), -# "category": fields.String(required=True), -# "subcategory": fields.String(required=True), -# "risk": fields.Integer(required=True), -# "tags": fields.List(fields.String()), -# "description": fields.String(), -# }, -# ) - # content filter profile class ContentFilterProfile(BaseModel): id: StrictStr @@ -216,30 +124,6 @@ class ContentFilterProfile(BaseModel): ignore_body: StrictBool -# m_contentfilterprofile = api.model( -# "Content Filter Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "ignore_alphanum": fields.Boolean(required=True), -# "args": fields.Raw(required=True), -# "headers": fields.Raw(required=True), -# "cookies": fields.Raw(required=True), -# "path": fields.Raw(required=True), -# "allsections": fields.Raw(), -# "decoding": fields.Raw(required=True), -# "masking_seed": fields.String(required=True), -# "content_type": fields.List(fields.String()), -# "active": fields.List(fields.String()), -# "report": fields.List(fields.String()), -# "ignore": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# "ignore_body": fields.Boolean(required=True), -# }, -# ) - # aclprofile class ACLProfile(BaseModel): id: StrictStr @@ -255,23 +139,6 @@ class ACLProfile(BaseModel): action: Optional[StrictStr] -# m_aclprofile = api.model( -# "ACL Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# }, -# ) - # Global Filter class GlobalFilter(BaseModel): id: StrictStr @@ -285,23 +152,9 @@ class GlobalFilter(BaseModel): rule: anyTypeUnion -# m_glbalfilter = api.model( -# "Global Filter", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "source": fields.String(required=True), -# "mdate": fields.String(required=True), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# "action": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# "rule": AnyType(), -# }, -# ) - # Flow Control + class FlowControl(BaseModel): id: StrictStr name: StrictStr @@ -315,25 +168,9 @@ class FlowControl(BaseModel): active: StrictBool -# -# m_flowcontrol = api.model( -# "Flow Control", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "timeframe": fields.Integer(required=True), -# "key": fields.List(fields.Raw(required=True)), -# "sequence": fields.List(fields.Raw(required=True)), -# "tags": fields.List(fields.String()), -# "include": fields.List(fields.String()), -# "exclude": fields.List(fields.String()), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# }, -# ) - # Action + class Action(BaseModel): id: StrictStr name: StrictStr @@ -343,22 +180,8 @@ class Action(BaseModel): type_: StrictStr = Field(alias="type") class Config: - fields = { - "_type": "type" - } - + fields = {"_type": "type"} -# m_action = api.model( -# "Action", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String(required=True)), -# "params": fields.Raw(), -# "type": fields.String(required=True), -# }, -# ) # Virtual Tag class VirtualTag(BaseModel): @@ -368,32 +191,12 @@ class VirtualTag(BaseModel): match: List[typing.Any] -# -# m_virtualtag = api.model( -# "Virtual Tag", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.List(fields.Raw(required=True)), -# }, -# ) - # custom class Custom(BaseModel): id: StrictStr name: StrictStr -# m_custom = api.model( -# "Custom", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - ### mapping from doc name to model models = { @@ -442,44 +245,8 @@ class DocumentMask(BaseModel): star_: Optional[List[typing.Any]] = Field(alias="*") class Config: - fields = { - "_type": "type" - } - + fields = {"_type": "type"} -# m_document_mask = api.model( -# "Mask for document", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(required=True), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# "include": fields.Wildcard(fields.Raw()), -# "exclude": fields.Wildcard(fields.Raw()), -# "tags": fields.List(fields.String()), -# "active": fields.Wildcard(fields.Raw()), -# "action": fields.Raw(), -# "sequence": fields.List(fields.Raw()), -# "timeframe": fields.Integer(), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "pairwith": fields.Raw(), -# "content_type": fields.List(fields.String()), -# "params": fields.Raw(), -# "decoding": fields.Raw(), -# "category": fields.String(), -# "subcategory": fields.String(), -# "risk": fields.Integer(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "match": fields.String(), -# "type": fields.String(), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) class VersionLog(BaseModel): version: Optional[StrictStr] @@ -488,16 +255,6 @@ class VersionLog(BaseModel): star_: Optional[List[typing.Any]] = Field(alias="*") -# -# m_version_log = api.model( -# "Version log", -# { -# "version": fields.String(), -# "date": fields.DateTime(dt_format="iso8601"), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - class Meta(BaseModel): id: StrictStr description: StrictStr @@ -506,54 +263,20 @@ class Meta(BaseModel): version: Optional[StrictStr] -# m_meta = api.model( -# "Meta", -# { -# "id": fields.String(required=True), -# "description": fields.String(required=True), -# "date": fields.DateTime(), -# "logs": fields.List(fields.Nested(m_version_log), default=[]), -# "version": fields.String(), -# }, -# ) - class BlobEntry(BaseModel): format: StrictStr blob: anyTypeUnion -# m_blob_entry = api.model( -# "Blob Entry", -# { -# "format": fields.String(required=True), -# "blob": AnyType(), -# }, -# ) - class BlobListEntry(BaseModel): name: Optional[StrictStr] -# m_blob_list_entry = api.model( -# "Blob ListEntry", -# { -# "name": fields.String(), -# }, -# ) - class DocumentListEntry(BaseModel): name: Optional[StrictStr] entries: Optional[StrictInt] -# m_document_list_entry = api.model( -# "Document ListEntry", -# { -# "name": fields.String(), -# "entries": fields.Integer(), -# }, -# ) - class ConfigDocuments(BaseModel): ratelimits: Optional[List[models["ratelimits"]]] = [] securitypolicies: Optional[List[models["securitypolicies"]]] = [] @@ -567,12 +290,6 @@ class ConfigDocuments(BaseModel): custom: Optional[List[models["custom"]]] = [] -# m_config_documents = api.model( -# "Config Documents", -# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -# ) - - class ConfigBlobs(BaseModel): geolite2asn: Optional[BlobEntry] geolite2country: Optional[BlobEntry] @@ -580,11 +297,6 @@ class ConfigBlobs(BaseModel): customconf: Optional[BlobEntry] -# m_config_blobs = api.model( -# "Config Blobs", -# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -# ) - class ConfigDeleteBlobs(BaseModel): geolite2asn: Optional[StrictBool] geolite2country: Optional[StrictBool] @@ -592,10 +304,6 @@ class ConfigDeleteBlobs(BaseModel): customconf: Optional[StrictBool] -# m_config_delete_blobs = api.model( -# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -# ) - class Config(BaseModel): meta: Meta = {} documents: ConfigDocuments = {} @@ -604,81 +312,37 @@ class Config(BaseModel): delete_blobs: ConfigDeleteBlobs = {} -# m_config = api.model( -# "Config", -# { -# "meta": fields.Nested(m_meta, default={}), -# "documents": fields.Nested(m_config_documents, default={}), -# "blobs": fields.Nested(m_config_blobs, default={}), -# "delete_documents": fields.Nested(m_config_documents, default={}), -# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), -# }, -# ) - class Edit(BaseModel): path: StrictStr value: StrictStr -# m_edit = api.model( -# "Edit", -# { -# "path": fields.String(required=True), -# "value": fields.String(required=True), -# }, -# ) - class BasicEntry(BaseModel): id: StrictStr name: StrictStr description: Optional[StrictStr] -# m_basic_entry = api.model( -# "Basic Document Entry", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# }, -# ) - ### Publish + class Bucket(BaseModel): name: StrictStr url: StrictStr -# m_bucket = api.model( -# "Bucket", -# { -# "name": fields.String(required=True), -# "url": fields.String(required=True), -# }, -# ) - ### Git push & pull + class GitUrl(BaseModel): giturl: StrictStr -# m_giturl = api.model( -# "GitUrl", -# { -# "giturl": fields.String(required=True), -# }, -# ) - ### Db class DB(BaseModel): pass -# m_db = api.model("db", {}) - - ### Document Schema validation @@ -729,7 +393,7 @@ def get_gitactor(request): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -740,7 +404,7 @@ def get_gitactor(request): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -777,7 +441,13 @@ class Tags(Enum): ### CONFIGS ### ################ -@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta], response_model_exclude_unset=True) + +@router.get( + "/configs/", + tags=[Tags.congifs], + response_model=List[Meta], + response_model_exclude_unset=True, +) async def configs_get(request: Request): """Get the detailed list of existing configurations""" res = request.app.backend.configs_list() @@ -791,20 +461,6 @@ async def configs_post(config: Config, request: Request): return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) -# -# @ns_configs.route("/") -# class Configs(Resource): -# # @ns_configs.marshal_list_with(m_meta, skip_none=True) -# # def get(self): -# # "Get the detailed list of existing configurations" -# # return current_app.backend.configs_list() -# -# @ns_configs.expect(m_config, validate=True) -# def post(self): -# "Create a new configuration" -# data = request.json -# return current_app.backend.configs_create(data, get_gitactor()) - @router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) async def config_get(config: str, request: Request): """Retrieve a complete configuration""" @@ -831,30 +487,6 @@ async def config_delete(config: str, request: Request): return request.app.backend.configs_delete(config) -# @ns_configs.route("//") -# class Config(Resource): -# # @ns_configs.marshal_with(m_config, skip_none=True) -# # def get(self, config): -# # "Retrieve a complete configuration" -# # return current_app.backend.configs_get(config) -# -# # @ns_configs.expect(m_config, validate=True) -# # def post(self, config): -# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" -# # data = request.json -# # return current_app.backend.configs_create(data, config, get_gitactor()) -# -# # @ns_configs.expect(m_meta, validate=True) -# # def put(self, config): -# # "Update an existing configuration" -# # data = request.json -# # return current_app.backend.configs_update(config, data, get_gitactor()) -# -# def delete(self, config): -# "Delete a configuration" -# return current_app.backend.configs_delete(config) - - @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" @@ -862,33 +494,18 @@ async def config_clone_post(config: str, meta: Meta, request: Request): return request.app.backend.configs_clone(config, data) -# @ns_configs.route("//clone/") -# class ConfigClone(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config): -# "Clone a configuration. New name is provided in POST data" -# data = request.json -# return current_app.backend.configs_clone(config, data) -# - @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): +async def config_clone_name_post( + config: str, new_name: str, meta: Meta, request: Request +): """Clone a configuration. New name is provided URL""" data = await request.json() return request.app.backend.configs_clone(config, data, new_name) - # @ns_configs.route("//clone//") - # class ConfigCloneName(Resource): - # @ns_configs.expect(m_meta, validate=True) - # def post(self, config, new_name): - # "Clone a configuration. New name is provided URL" - # data = request.json - # return current_app.backend.configs_clone(config, data, new_name) - def filter_x_fields(res, x_fields): fields = [] - if x_fields.startswith(('[', '{', '(')): + if x_fields.startswith(("[", "{", "(")): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") @@ -898,8 +515,12 @@ def filter_x_fields(res, x_fields): return {field: res[field] for field in fields if field in res} -@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) +@router.get( + "/configs/{config}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" res = request.app.backend.configs_list_versions(config) @@ -908,67 +529,44 @@ async def config_list_version_get(config: str, request: Request): return res -# @ns_configs.route("//v/") -# class ConfigListVersion(Resource): -# @ns_configs.marshal_with(m_version_log, skip_none=True) -# def get(self, config): -# "Get all versions of a given configuration" -# return current_app.backend.configs_list_versions(config) - - @router.get("/configs/{config}/v/{version}/", tags=[Tags.congifs]) async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" return request.app.backend.configs_get(config, version) -# @ns_configs.route("//v//") -# class ConfigVersion(Resource): -# def get(self, config, version): -# "Retrieve a specific version of a configuration" -# return current_app.backend.configs_get(config, version) - @router.get("/configs/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" return request.app.backend.configs_revert(config, version, get_gitactor(request)) -# @ns_configs.route("//v//revert/") -# class ConfigRevert(Resource): -# def put(self, config, version): -# "Create a new version for a configuration from an old version" -# return current_app.backend.configs_revert(config, version, get_gitactor()) - - ############# ### Blobs ### ############# -@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry]) +@router.get( + "/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry] +) async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" res = request.app.backend.blobs_list(config) return res -# @ns_configs.route("//b/") -# class BlobsResource(Resource): -# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of available blobs" -# res = current_app.backend.blobs_list(config) -# return res - -@router.get("/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +@router.get( + "/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry +) async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" return request.app.backend.blobs_get(config, blob) @router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): +async def blob_resource_post( + config: str, blob: str, blob_entry: BlobEntry, request: Request +): """Create a new blob""" b_entry = await request.json() return request.app.backend.blobs_create( @@ -977,7 +575,9 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ @router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): +async def blob_resource_put( + config: str, blob: str, blob_entry: BlobEntry, request: Request +): """upaate an existing blob""" b_entry = await request.json() @@ -992,35 +592,12 @@ async def blob_resource_delete(config: str, blob: str, request: Request): return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) -# -# @ns_configs.route("//b//") -# class BlobResource(Resource): -# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) -# # def get(self, config, blob): -# # "Retrieve a blob" -# # return current_app.backend.blobs_get(config, blob) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def post(self, config, blob): -# # "Create a new blob" -# # return current_app.backend.blobs_create( -# # config, blob, request.json, get_gitactor() -# # ) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def put(self, config, blob): -# # "Replace a blob with new data" -# # return current_app.backend.blobs_update( -# # config, blob, request.json, get_gitactor() -# # ) -# -# def delete(self, config, blob): -# "Delete a blob" -# return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) +@router.get( + "/configs/{config}/b/{blob}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) async def blob_list_version_resource_get(config: str, blob: str, request: Request): """Retrieve the list of versions of a given blob""" res = request.app.backend.blobs_list_versions(config, blob) @@ -1029,17 +606,14 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques return res -# @ns_configs.route("//b//v/") -# class BlobListVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob): -# "Retrieve the list of versions of a given blob" -# res = current_app.backend.blobs_list_versions(config, blob) -# return res - - -@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): +@router.get( + "/configs/{config}/b/{blob}/v/{version}/", + tags=[Tags.congifs], + response_model=BlobEntry, +) +async def blob_version_resource_get( + config: str, blob: str, version: str, request: Request +): """Retrieve the given version of a blob""" res = request.app.backend.blobs_get(config, blob, version) @@ -1048,95 +622,73 @@ async def blob_version_resource_get(config: str, blob: str, version: str, reques return request.app.backend.blobs_get(config, blob, version) -# @ns_configs.route("//b//v//") -# class BlobVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob, version): -# "Retrieve the given version of a blob" -# return current_app.backend.blobs_get(config, blob, version) - @router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) -async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): +async def blob_revert_resource_put( + config: str, blob: str, version: str, request: Request +): """Create a new version for a blob from an old version""" - return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) - - -# -# @ns_configs.route("//b//v//revert/") -# class BlobRevertResource(Resource): -# def put(self, config, blob, version): -# "Create a new version for a blob from an old version" -# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) + return request.app.backend.blobs_revert( + config, blob, version, get_gitactor(request) + ) ################# ### DOCUMENTS ### ################# -@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) + +@router.get( + "/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry] +) async def document_resource_get(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) return res -# -# @ns_configs.route("//d/") -# class DocumentsResource(Resource): -# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of existing documents in this configuration" -# res = current_app.backend.documents_list(config) -# return res - - @router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_get(config: str, document: str, request: Request): def filter_document_mask(res, x_fields): fields = [] if x_fields: - if x_fields.startswith(('[', '{', '(')): + if x_fields.startswith(("[", "{", "(")): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") else: fields = switch_alias(DocumentMask.__fields__.keys()) - print(f"fields are {fields}") - print(f"res is {res}") - return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" headers = request.headers - print(headers) if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - # res = {key: res[key] for key in list(models[document].__fields__.keys())} return filter_document_mask(res, headers.get("x-fields", None)) async def _filter(data, keys): - # filtered = {} - # for key in keys: - # if data.get(key, False): - # filtered[key] = data[key] - # return filtered return {key: data[key] for key in switch_alias(keys) if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): +async def document_resource_post( + config: str, document: str, basic_entries: List[BasicEntry], request: Request +): """Create a new complete document""" if document not in models.keys(): - raise HTTPException(status_code=404, detail="document name is not one of the possible name in 'models' module") + raise HTTPException( + status_code=404, + detail="document name is not one of the possible name in 'models' module", + ) as_dict = await request.json() - print(as_dict[0]) - data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] - print(data[0]) + data = [ + await _filter(dict(entry), list(models[document].__fields__.keys())) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: @@ -1148,16 +700,23 @@ async def document_resource_post(config: str, document: str, basic_entries: List @router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): +async def document_resource_put( + config: str, document: str, basic_entries: List[BasicEntry], request: Request +): """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) for entry in as_dict] + data = [ + await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: - raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + raise HTTPException( + 500, "schema mismatched for entry: " + str(entry) + "\n" + err + ) res = request.app.backend.documents_update( config, document, data, get_gitactor(request) ) @@ -1173,56 +732,14 @@ async def document_resource_delete(config: str, document: str, request: Request) return res -# @ns_configs.route("//d//") -# class DocumentResource(Resource): -# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) -# # def get(self, config, document): -# # "Get a complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_get(config, document) -# # return marshal(res, models[document], skip_none=True) -# # -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def post(self, config, document): -# # "Create a new complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched: \n" + err) -# # res = current_app.backend.documents_create( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def put(self, config, document): -# # "Update an existing document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) -# # res = current_app.backend.documents_update( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # def delete(self, config, document): -# # "Delete/empty a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_delete(config, document, get_gitactor()) -# # return res - - -@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) -async def document_list_version_resource_get(config: str, document: str, request: Request): +@router.get( + "/configs/{config}/d/{document}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], +) +async def document_list_version_resource_get( + config: str, document: str, request: Request +): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1231,56 +748,39 @@ async def document_list_version_resource_get(config: str, document: str, request return res -# -# @ns_configs.route("//d//v/") -# class DocumentListVersionResource(Resource): -# def get(self, config, document): -# "Retrieve the existing versions of a given document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_list_versions(config, document) -# return marshal(res, m_version_log, skip_none=True) - - @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str, request: Request): +async def document_version_resource_get( + config: str, document: str, version: str, request: Request +): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in r} for r in res] - + return [ + { + key: r[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in r + } + for r in res + ] -# @ns_configs.route("//d//v//") -# class DocumentVersionResource(Resource): -# def get(self, config, document, version): -# "Get a given version of a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_get(config, document, version) -# return marshal(res, models[document], skip_none=True) @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) -async def document_revert_resource_put(config: str, document: str, version: str, request: Request): +async def document_revert_resource_put( + config: str, document: str, version: str, request: Request +): """Create a new version for a document from an old version""" return request.app.backend.documents_revert( config, document, version, get_gitactor(request) ) -# @ns_configs.route("//d//v//revert/") -# class DocumentRevertResource(Resource): -# def put(self, config, document, version): -# "Create a new version for a document from an old version" -# return current_app.backend.documents_revert( -# config, document, version, get_gitactor() -# ) - - ############### ### ENTRIES ### ############### + @router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_get(config: str, document: str, request: Request): """Retrieve the list of entries in a document""" @@ -1291,7 +791,9 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): +async def entries_resource_post( + config: str, document: str, basic_entry: BasicEntry, request: Request +): "Create an entry in a document" data_json = await request.json() @@ -1309,31 +811,6 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(500, "schema mismatched: \n" + err) -# @ns_configs.route("//d//e/") -# class EntriesResource(Resource): -# # def get(self, config, document): -# # "Retrieve the list of entries in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_list(config, document) -# # return res # XXX: marshal -# -# @ns_configs.expect(m_basic_entry, validate=True) -# def post(self, config, document): -# "Create an entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# isValid, err = validateJson(request.json, document) -# if isValid: -# data = marshal(request.json, models[document], skip_none=True) -# res = current_app.backend.entries_create( -# config, document, data, get_gitactor() -# ) -# return res -# else: -# abort(500, "schema mismatched: \n" + err) - - def switch_alias(keys): return [key[:-1] if key.endswith("_") else key for key in keys] @@ -1349,15 +826,20 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): +async def entry_resource_put( + config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request +): """Update an entry in a document""" data_json = await request.json() - print(f"-------------------------- active: {data_json.get('active', 'blaaaaaaa')} ------------------------------") if document not in models: raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in data_json} + data = { + key: data_json[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in data_json + } res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1368,7 +850,9 @@ async def entry_resource_put(config: str, document: str, entry: str, basic_entry @router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): +async def entry_resource_deleye( + config: str, document: str, entry: str, request: Request +): """Delete an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1378,65 +862,31 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: return res -# @ns_configs.route("//d//e//") -# class EntryResource(Resource): -# # def get(self, config, document, entry): -# # "Retrieve an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_get(config, document, entry) -# # return marshal(res, models[document], skip_none=True) -# -# # @ns_configs.expect(m_basic_entry, validate=True) -# # def put(self, config, document, entry): -# # "Update an entry in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # isValid, err = validateJson(request.json, document) -# # if isValid: -# # data = marshal(request.json, models[document], skip_none=True) -# # res = current_app.backend.entries_update( -# # config, document, entry, data, get_gitactor() -# # ) -# # return res -# # else: -# # abort(500, "schema mismatched: \n" + err) -# -# # def delete(self, config, document, entry): -# # "Delete an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_delete( -# # config, document, entry, get_gitactor() -# # ) -# # return res - - -@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) -async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request, - ): +@router.get( + "/configs/{config}/d/{document}/e/{entry}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) +async def entry_list_version_resource_get( + config: str, + document: str, + entry: str, + request: Request, +): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_list_versions(config, document, entry) return res - # return {key: res[key] for key in list(VersionLog.__fields__.keys())} - - -# -# @ns_configs.route("//d//e//v/") -# class EntryListVersionResource(Resource): -# def get(self, config, document, entry): -# "Get the list of existing versions of a given entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_list_versions(config, document, entry) -# return marshal(res, m_version_log, skip_none=True) -@router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str, request: Request): +@router.get( + "/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs] +) +async def entry_version_resource_get( + config: str, document: str, entry: str, version: str, request: Request +): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1445,49 +895,23 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver return {key: res[key] for key in keys if key in res} -# @ns_configs.route( -# "//d//e//v//" -# ) -# class EntryVersionResource(Resource): -# def get(self, config, document, entry, version): -# "Get a given version of a document entry" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_get(config, document, entry, version) -# return marshal(res, models[document], skip_none=True) - - ################ ### Database ### ################ + @router.get("/db/", tags=[Tags.db]) async def db_resource_get(request: Request): """Get the list of existing namespaces""" return request.app.backend.ns_list() -# @ns_db.route("/") -# class DbResource(Resource): -# def get(self): -# "Get the list of existing namespaces" -# return current_app.backend.ns_list() - - @router.get("/db/v/", tags=[Tags.db]) async def db_query_resource_get(request: Request): """List all existing versions of namespaces""" return request.app.backend.ns_list_versions() -# -# @ns_db.route("/v/") -# class DbQueryResource(Resource): -# def get(self): -# "List all existing versions of namespaces" -# return current_app.backend.ns_list_versions() - - @router.get("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_get(nsname: str, request: Request): """Get a complete namespace""" @@ -1524,67 +948,21 @@ async def ns_resource_put(nsname: str, request: Request): raise HTTPException(409, "namespace [%s] does not exist" % nsname) -# @ns_db.route("//") -# class NSResource(Resource): -# # def get(self, nsname): -# # "Get a complete namespace" -# # try: -# # return current_app.backend.ns_get(nsname, version=None) -# # except KeyError: -# # abort(404, "namespace [%s] does not exist" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def post(self, nsname): -# # "Create a non-existing namespace from data" -# # try: -# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) -# # except Exception: -# # abort(409, "namespace [%s] already exists" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def put(self, nsname): -# # "Merge data into a namespace" -# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) -# -# # def delete(self, nsname): -# # "Delete an existing namespace" -# # try: -# # return current_app.backend.ns_delete(nsname, get_gitactor()) -# # except KeyError: -# # abort(409, "namespace [%s] does not exist" % nsname) - - @router.get("/db/{nsname}/v/{version}/", tags=[Tags.db]) async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" return request.app.backend.ns_get(nsname, version) -# @ns_db.route("//v//") -# class NSVersionResource(Resource): -# def get(self, nsname, version): -# "Get a given version of a namespace" -# return current_app.backend.ns_get(nsname, version) - - @router.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): """Create a new version for a namespace from an old version""" try: return request.app.backend.ns_revert(nsname, version, get_gitactor(request)) except KeyError: - raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -# -# @ns_db.route("//v//revert/") -# class NSVersionResource(Resource): -# def put(self, nsname, version): -# "Create a new version for a namespace from an old version" -# try: -# return current_app.backend.ns_revert(nsname, version, get_gitactor()) -# except KeyError: -# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + raise HTTPException( + 404, "namespace [%s] version [%s] not found" % (nsname, version) + ) @router.post("/db/{nsname}/q/", tags=[Tags.db]) @@ -1594,39 +972,18 @@ async def ns_query_resource_post(nsname: str, request: Request): return request.app.backend.ns_query(nsname, req_json) -# @ns_db.route("//q/") -# class NSQueryResource(Resource): -# def post(self, nsname): -# "Run a JSON query on the namespace and returns the results" -# return current_app.backend.ns_query(nsname, request.json) -# - - @router.get("/db/{nsname}/k/", tags=[Tags.db]) async def keys_resource_get(nsname: str, request: Request): """List all keys of a given namespace""" return request.app.backend.key_list(nsname) -# @ns_db.route("//k/") -# class KeysResource(Resource): -# def get(self, nsname): -# "List all keys of a given namespace" -# return current_app.backend.key_list(nsname) - @router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) async def keys_list_versions_resource_get(nsname: str, key: str, request: Request): """Get all versions of a given key in namespace""" return request.app.backend.key_list_versions(nsname, key) -# @ns_db.route("//k//v/") -# class KeysListVersionsResource(Resource): -# def get(self, nsname, key): -# "Get all versions of a given key in namespace" -# return current_app.backend.key_list_versions(nsname, key) -# - @router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) async def key_resource_get(nsname: str, key: str, request: Request): """Retrieve a given key's value from a given namespace""" @@ -1661,35 +1018,6 @@ async def key_resource_delete(nsname: str, key: str, request: Request): return request.app.backend.key_delete(nsname, key, get_gitactor(request)) -# @ns_db.route("//k//") -# class KeyResource(Resource): -# # def get(self, nsname, key): -# # "Retrieve a given key's value from a given namespace" -# # return current_app.backend.key_get(nsname, key) -# -# # def put(self, nsname, key): -# # "Create or update the value of a key" -# # # check if "reblaze/k/" exists in system/schema-validation -# # if nsname != "system": -# # keyName = nsname + "/k/" + key -# # schemas = current_app.backend.key_get("system", "schema-validation") -# # schema = None -# # # find schema if exists and validate the json input -# # for item in schemas.items(): -# # if item[0] == keyName: -# # schema = item[1] -# # break -# # if schema: -# # isValid = validateDbJson(request.json, schema) -# # if isValid is False: -# # abort(500, "schema mismatched") -# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) -# -# # def delete(self, nsname, key): -# # "Delete a key" -# # return current_app.backend.key_delete(nsname, key, get_gitactor()) - - ############# ### Tools ### ############# @@ -1705,22 +1033,11 @@ async def fetch_resource_get(url: str): return r.content -# @ns_tools.route("/fetch") -# class FetchResource(Resource): -# @ns_tools.expect(req_fetch_parser, validate=True) -# def get(self): -# "Fetch an URL" -# args = req_fetch_parser.parse_args() -# try: -# r = requests.get(args.url) -# except Exception as e: -# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) -# return make_response(r.content) - - @router.put("/tools/publish/{config}/", tags=[Tags.tools]) @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) -async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): +async def publish_resource_put( + config: str, request: Request, buckets: List[Bucket], version: str = None +): """Push configuration to s3 buckets""" conf = request.app.backend.configs_get(config, version) ok = True @@ -1740,40 +1057,10 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck else: s = True msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) + status.append({"name": bucket["name"], "ok": s, "logs": logs, "message": msg}) return {"ok": ok, "status": status} -# @ns_tools.route("/publish//") -# @ns_tools.route("/publish//v//") -# class PublishResource(Resource): -# @ns_tools.expect([m_bucket], validate=True) -# def put(self, config, version=None): -# "Push configuration to s3 buckets" -# conf = current_app.backend.configs_get(config, version) -# ok = True -# status = [] -# if type(request.json) is not list: -# abort(400, "body must be a list") -# for bucket in request.json: -# logs = [] -# try: -# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) -# except Exception as e: -# ok = False -# s = False -# msg = repr(e) -# else: -# s = True -# msg = "ok" -# status.append( -# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} -# ) -# return make_response({"ok": ok, "status": status}) - - @router.put("/tools/gitpush/", tags=[Tags.tools]) async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" @@ -1793,26 +1080,6 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): return {"ok": ok, "status": status} -# @ns_tools.route("/gitpush/") -# class GitPushResource(Resource): -# @ns_tools.expect([m_giturl], validate=True) -# def put(self): -# "Push git configuration to remote git repositories" -# ok = True -# status = [] -# for giturl in request.json: -# try: -# current_app.backend.gitpush(giturl["giturl"]) -# except Exception as e: -# msg = repr(e) -# s = False -# else: -# msg = "ok" -# s = True -# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) -# return make_response({"ok": ok, "status": status}) - - @router.put("/tools/gitfetch/", tags=[Tags.tools]) async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" @@ -1826,19 +1093,3 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): else: msg = "ok" return {"ok": ok, "status": msg} - - -# @ns_tools.route("/gitfetch/") -# class GitFetchResource(Resource): -# @ns_tools.expect(m_giturl, validate=True) -# def put(self): -# "Fetch git configuration from specified remote repository" -# ok = True -# try: -# current_app.backend.gitfetch(request.json["giturl"]) -# except Exception as e: -# ok = False -# msg = repr(e) -# else: -# msg = "ok" -# return make_response({"ok": ok, "status": msg}) \ No newline at end of file diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index e57e5578e..d5a5fee86 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -36,7 +36,7 @@ "fastapi==0.87.0", "prometheus-fastapi-instrumentator==5.9.1", "pydantic==1.10.2", - "uvicorn==0.19.0" + "uvicorn==0.19.0", ], classifiers=[ "Programming Language :: Python :: 3", From 9657dff633585228fd01a173e59e70f3550ac91c Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 16:17:37 +0200 Subject: [PATCH 22/55] delete temp api file --- .../curieconf/confserver/v3/fast_api.py | 1773 ----------------- 1 file changed, 1773 deletions(-) delete mode 100644 curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py deleted file mode 100644 index 076af7992..000000000 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ /dev/null @@ -1,1773 +0,0 @@ -import datetime -import typing -from enum import Enum -from typing import Optional, List, Union - -from fastapi import FastAPI, Request, HTTPException -from pydantic import BaseModel, Field, validator -import random # needed for generating a random number for an API -import uvicorn # optional if you run it directly from terminal -import jsonschema - -# monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type -jsonschema.Draft4Validator = jsonschema.Draft3Validator - -# from curieconf import utils -from curieconf.utils import cloud -# from curieconf.confserver import app - - -# TODO: TEMP DEFINITIONS -import os - -app = FastAPI() -app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") -options = {} -val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) -if val: - options["trusted_username_header"] = val -val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) -if val: - options["trusted_email_header"] = val - -import requests -from jsonschema import validate -from pathlib import Path -import json - -# api_bp = Blueprint("api_v3", __name__) -# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - -# ns_configs = api.namespace("configs", description="Configurations") -# ns_db = api.namespace("db", description="Database") -# ns_tools = api.namespace("tools", description="Tools") - - -############## -### MODELS ### -############## - - -### Models for documents -anyTypeUnion = Union[int, float, bool, object, list, None] -anyOp = Optional[object] -anyType = ["number", "string", "boolean", "object", "array", "null"] - - -# class AnyType(fields.Raw): -# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] - - -# limit - -class Threshold(BaseModel): - limit: int - action: str - - -# m_threshold = api.model( -# "Rate Limit Threshold", -# { -# "limit": fields.Integer(required=True), -# "action": fields.String(required=True), -# }, -# ) - -class Limit(BaseModel): - id: str - name: str - description: Optional[str] - _global: bool = Field(alias="global") - active: bool - timeframe: int - thresholds: List[Threshold] - include: typing.Any - exclude: typing.Any - key: anyTypeUnion - pairwith: typing.Any - tags: List[str] - - -# m_limit = api.model( -# "Rate Limit", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "global": fields.Boolean(required=True), -# "active": fields.Boolean(required=True), -# "timeframe": fields.Integer(required=True), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "include": fields.Raw(required=True), -# "exclude": fields.Raw(required=True), -# "key": AnyType(required=True), -# "pairwith": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# }, -# ) - -# securitypolicy -class SecProfileMap(BaseModel): - id: str - name: str - description: str - match: str - acl_profile: str - acl_active: bool - content_filter_profile: str - content_filter_active: bool - limit_ids: Optional[list] - - -# m_secprofilemap = api.model( -# "Security Profile Map", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.String(required=True), -# "acl_profile": fields.String(required=True), -# "acl_active": fields.Boolean(required=True), -# "content_filter_profile": fields.String(required=True), -# "content_filter_active": fields.Boolean(required=True), -# "limit_ids": fields.List(fields.Raw()), -# }, -# ) - -# TODO = deprecated? -# m_map = api.model( -# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -# ) - -class SecurityPolicy(BaseModel): - id: str - name: str - description: str - tags: List[str] - match: str - session: anyTypeUnion - session_ids: anyTypeUnion - map: List[SecProfileMap] - - -# m_securitypolicy = api.model( -# "Security Policy", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String()), -# "match": fields.String(required=True), -# "session": AnyType(), -# "session_ids": AnyType(), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# }, -# ) - -# content filter rule - -class ContentFilterRule(BaseModel): - id: str - name: str - msg: str - operand: str - severity: int - certainity: int - category: str - subcategory: str - risk: int - tags: Optional[List[str]] - description: Optional[str] - - -# m_contentfilterrule = api.model( -# "Content Filter Rule", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "msg": fields.String(required=True), -# "operand": fields.String(required=True), -# "severity": fields.Integer(required=True), -# "certainity": fields.Integer(required=True), -# "category": fields.String(required=True), -# "subcategory": fields.String(required=True), -# "risk": fields.Integer(required=True), -# "tags": fields.List(fields.String()), -# "description": fields.String(), -# }, -# ) - -# content filter profile -class ContentFilterProfile(BaseModel): - id: str - name: str - description: Optional[str] - ignore_alphanum: bool - args: typing.Any - headers: typing.Any - cookies: typing.Any - path: typing.Any - allsections: typing.Any - decoding: typing.Any - masking_seed: str - content_type: Optional[List[str]] - active: Optional[List[str]] - report: Optional[List[str]] - ignore: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - ignore_body: bool - - -# m_contentfilterprofile = api.model( -# "Content Filter Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "ignore_alphanum": fields.Boolean(required=True), -# "args": fields.Raw(required=True), -# "headers": fields.Raw(required=True), -# "cookies": fields.Raw(required=True), -# "path": fields.Raw(required=True), -# "allsections": fields.Raw(), -# "decoding": fields.Raw(required=True), -# "masking_seed": fields.String(required=True), -# "content_type": fields.List(fields.String()), -# "active": fields.List(fields.String()), -# "report": fields.List(fields.String()), -# "ignore": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# "ignore_body": fields.Boolean(required=True), -# }, -# ) - -# aclprofile -class ACLProfile(BaseModel): - id: str - name: str - description: Optional[str] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - - -# m_aclprofile = api.model( -# "ACL Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# }, -# ) - -# Global Filter -class GlobalFilter(BaseModel): - id: str - name: str - source: str - mdate: str - description: str - active: bool - action: typing.Any - tags: Optional[List[str]] - rule: anyTypeUnion - - -# m_glbalfilter = api.model( -# "Global Filter", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "source": fields.String(required=True), -# "mdate": fields.String(required=True), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# "action": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# "rule": AnyType(), -# }, -# ) - -# Flow Control - -class FlowControl(BaseModel): - id: str - name: str - timeframe: int - key: List[typing.Any] - sequence: List[typing.Any] - tags: Optional[List[str]] - include: Optional[List[str]] - exclude: Optional[List[str]] - description: Optional[str] - active: bool - - -# -# m_flowcontrol = api.model( -# "Flow Control", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "timeframe": fields.Integer(required=True), -# "key": fields.List(fields.Raw(required=True)), -# "sequence": fields.List(fields.Raw(required=True)), -# "tags": fields.List(fields.String()), -# "include": fields.List(fields.String()), -# "exclude": fields.List(fields.String()), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# }, -# ) - -# Action - -class Action(BaseModel): - id: str - name: str - description: Optional[str] - tags: List[str] - params: typing.Any - type: str - - -# m_action = api.model( -# "Action", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String(required=True)), -# "params": fields.Raw(), -# "type": fields.String(required=True), -# }, -# ) - -# Virtual Tag -class VirtualTag(BaseModel): - id: str - name: str - description: Optional[str] - match: List[typing.Any] - - -# -# m_virtualtag = api.model( -# "Virtual Tag", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.List(fields.Raw(required=True)), -# }, -# ) - -# custom -class Custom(BaseModel): - id: str - name: str - - -# m_custom = api.model( -# "Custom", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -### mapping from doc name to model - -models = { - "ratelimits": Limit, - "securitypolicies": SecurityPolicy, - "contentfilterrules": ContentFilterRule, - "contentfilterprofiles": ContentFilterProfile, - "aclprofiles": ACLProfile, - "globalfilters": GlobalFilter, - "flowcontrol": FlowControl, - "actions": Action, - "virtualtags": Custom, - "custom": Custom, -} - - -### Other models -class DocumentMask(BaseModel): - id: str - name: str - description: str - map: Optional[List[SecProfileMap]] - include: Optional[List[typing.Any]] - exclude: Optional[List[typing.Any]] - tags: Optional[List[str]] - active: Optional[List[typing.Any]] - action: typing.Any - sequence: Optional[List[typing.Any]] - timeframe: Optional[int] - thresholds: Optional[List[Threshold]] - pairwith: typing.Any - content_type: Optional[List[str]] - params: typing.Any - decoding: typing.Any - category: Optional[str] - subcategory: Optional[str] - risk: Optional[int] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - match: Optional[str] = "j" - _type: Optional[str] = Field(alias="type") - _star: Optional[List[typing.Any]] = Field(alias="*") - - -# m_document_mask = api.model( -# "Mask for document", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(required=True), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# "include": fields.Wildcard(fields.Raw()), -# "exclude": fields.Wildcard(fields.Raw()), -# "tags": fields.List(fields.String()), -# "active": fields.Wildcard(fields.Raw()), -# "action": fields.Raw(), -# "sequence": fields.List(fields.Raw()), -# "timeframe": fields.Integer(), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "pairwith": fields.Raw(), -# "content_type": fields.List(fields.String()), -# "params": fields.Raw(), -# "decoding": fields.Raw(), -# "category": fields.String(), -# "subcategory": fields.String(), -# "risk": fields.Integer(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "match": fields.String(), -# "type": fields.String(), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -class VersionLog(BaseModel): - version: Optional[str] - # TODO - dt_format="iso8601" - date: Optional[datetime.datetime] - _star: Optional[List[typing.Any]] = Field(alias="*") - - -# -# m_version_log = api.model( -# "Version log", -# { -# "version": fields.String(), -# "date": fields.DateTime(dt_format="iso8601"), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -class Meta(BaseModel): - id: str - description: str - date: Optional[datetime.datetime] - logs: Optional[List[VersionLog]] = [] - version: Optional[str] - - -# m_meta = api.model( -# "Meta", -# { -# "id": fields.String(required=True), -# "description": fields.String(required=True), -# "date": fields.DateTime(), -# "logs": fields.List(fields.Nested(m_version_log), default=[]), -# "version": fields.String(), -# }, -# ) - -class BlobEntry(BaseModel): - format: str - blob: anyTypeUnion - - -# m_blob_entry = api.model( -# "Blob Entry", -# { -# "format": fields.String(required=True), -# "blob": AnyType(), -# }, -# ) - -class BlobListEntry(BaseModel): - name: Optional[str] - - -# m_blob_list_entry = api.model( -# "Blob ListEntry", -# { -# "name": fields.String(), -# }, -# ) - -class DocumentListEntry(BaseModel): - name: Optional[str] - entries: Optional[int] - - -# m_document_list_entry = api.model( -# "Document ListEntry", -# { -# "name": fields.String(), -# "entries": fields.Integer(), -# }, -# ) - -class ConfigDocuments(BaseModel): - ratelimits: Optional[List[models["ratelimits"]]] = [] - securitypolicies: Optional[List[models["securitypolicies"]]] = [] - contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] - contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] - aclprofiles: Optional[List[models["aclprofiles"]]] = [] - globalfilters: Optional[List[models["globalfilters"]]] = [] - flowcontrol: Optional[List[models["flowcontrol"]]] = [] - actions: Optional[List[models["actions"]]] = [] - virtualtags: Optional[List[models["virtualtags"]]] = [] - custom: Optional[List[models["custom"]]] = [] - - -# m_config_documents = api.model( -# "Config Documents", -# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -# ) - - -class ConfigBlobs(BaseModel): - geolite2asn: Optional[List[Optional[BlobEntry]]] - geolite2country: Optional[List[Optional[BlobEntry]]] - geolite2city: Optional[List[Optional[BlobEntry]]] - customconf: Optional[List[Optional[BlobEntry]]] - - -# m_config_blobs = api.model( -# "Config Blobs", -# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -# ) - -class ConfigDeleteBlobs(BaseModel): - geolite2asn: Optional[bool] - geolite2country: Optional[bool] - geolite2city: Optional[bool] - customconf: Optional[bool] - - -# m_config_delete_blobs = api.model( -# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -# ) - -class Config(BaseModel): - meta: Meta = {} - documents: ConfigDocuments = {} - blobs: ConfigBlobs = {} - delete_documents: ConfigDocuments = {} - delete_blobs: ConfigDeleteBlobs = {} - - -# m_config = api.model( -# "Config", -# { -# "meta": fields.Nested(m_meta, default={}), -# "documents": fields.Nested(m_config_documents, default={}), -# "blobs": fields.Nested(m_config_blobs, default={}), -# "delete_documents": fields.Nested(m_config_documents, default={}), -# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), -# }, -# ) - -class Edit(BaseModel): - path: str - value: str - - -# m_edit = api.model( -# "Edit", -# { -# "path": fields.String(required=True), -# "value": fields.String(required=True), -# }, -# ) - -class BasicEntry(BaseModel): - id: str - name: str - description: Optional[str] - - -# m_basic_entry = api.model( -# "Basic Document Entry", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# }, -# ) - -### Publish - -class Bucket(BaseModel): - name: str - url: str - - -# m_bucket = api.model( -# "Bucket", -# { -# "name": fields.String(required=True), -# "url": fields.String(required=True), -# }, -# ) - -### Git push & pull - -class GitUrl(BaseModel): - giturl: str - - -# m_giturl = api.model( -# "GitUrl", -# { -# "giturl": fields.String(required=True), -# }, -# ) - -### Db -class DB(BaseModel): - pass - - -# m_db = api.model("db", {}) - - -### Document Schema validation - - -def validateJson(json_data, schema_type): - try: - validate(instance=json_data, schema=schema_type_map[schema_type]) - except jsonschema.exceptions.ValidationError as err: - print(str(err)) - return False, str(err) - return True, "" - - -### DB Schema validation - - -def validateDbJson(json_data, schema): - try: - validate(instance=json_data, schema=schema) - except jsonschema.exceptions.ValidationError as err: - print(str(err)) - return False - return True - - -### Set git actor according to config & defined HTTP headers - - -def get_gitactor(request): - email, username = "", "" - email_header = app.options.get("trusted_email_header", None) - if email_header: - email = request.headers.get(email_header, "") - username_header = app.options.get("trusted_username_header", None) - if username_header: - username = request.headers.get(username_header, "") - return app.backend.prepare_actor(username, email) - - -base_path = Path(__file__).parent -# base_path = "/etc/curiefense/json/" -acl_profile_file_path = (base_path / "./json/acl-profile.schema").resolve() -with open(acl_profile_file_path) as json_file: - acl_profile_schema = json.load(json_file) -ratelimits_file_path = (base_path / "./json/rate-limits.schema").resolve() -with open(ratelimits_file_path) as json_file: - ratelimits_schema = json.load(json_file) -securitypolicies_file_path = (base_path / "./json/security-policies.schema").resolve() -with open(securitypolicies_file_path) as json_file: - securitypolicies_schema = json.load(json_file) -content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" -).resolve() -with open(content_filter_profile_file_path) as json_file: - content_filter_profile_schema = json.load(json_file) -globalfilters_file_path = (base_path / "./json/global-filters.schema").resolve() -with open(globalfilters_file_path) as json_file: - globalfilters_schema = json.load(json_file) -flowcontrol_file_path = (base_path / "./json/flow-control.schema").resolve() -with open(flowcontrol_file_path) as json_file: - flowcontrol_schema = json.load(json_file) -content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" -).resolve() -with open(content_filter_rule_file_path) as json_file: - content_filter_rule_schema = json.load(json_file) -action_file_path = (base_path / "./json/action.schema").resolve() -with open(action_file_path) as json_file: - action_schema = json.load(json_file) -virtualtag_file_path = (base_path / "./json/virtual-tags.schema").resolve() -with open(virtualtag_file_path) as json_file: - virtual_tags_schema = json.load(json_file) -custom_file_path = (base_path / "./json/custom.schema").resolve() -with open(custom_file_path) as json_file: - custom_schema = json.load(json_file) -schema_type_map = { - "ratelimits": ratelimits_schema, - "securitypolicies": securitypolicies_schema, - "contentfilterprofiles": content_filter_profile_schema, - "aclprofiles": acl_profile_schema, - "globalfilters": globalfilters_schema, - "flowcontrol": flowcontrol_schema, - "contentfilterrules": content_filter_rule_schema, - "actions": action_schema, - "virtualtags": virtual_tags_schema, - "custom": custom_schema, -} - - -class Tags(Enum): - congifs = "configs" - db = "db" - tools = "tools" - - -################ -### CONFIGS ### -################ - -@app.get("/configs/", tags=[Tags.congifs], response_model=Meta) -async def configs_get(): - """Get the detailed list of existing configurations""" - return app.backend.configs_list() - - -@app.post("/configs/", tags=[Tags.congifs]) -async def configs_post(config: Config, request: Request): - """Create a new configuration""" - data = dict(config) - return app.backend.configs_create(data, get_gitactor(request)) - - -# -# @ns_configs.route("/") -# class Configs(Resource): -# # @ns_configs.marshal_list_with(m_meta, skip_none=True) -# # def get(self): -# # "Get the detailed list of existing configurations" -# # return current_app.backend.configs_list() -# -# @ns_configs.expect(m_config, validate=True) -# def post(self): -# "Create a new configuration" -# data = request.json -# return current_app.backend.configs_create(data, get_gitactor()) - -@app.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) -async def config_get(config: str): - """Retrieve a complete configuration""" - return app.backend.configs_get(config) - - -@app.post("/configs/{config}/", tags=[Tags.congifs]) -async def config_post(config: str, m_config: Config, request: Request): - "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = dict(m_config) - return app.backend.configs_create(data, config, get_gitactor(request)) - - -@app.put("/configs/{config}/", tags=[Tags.congifs]) -async def config_put(config: str, meta: Meta, request: Request): - """Update an existing configuration""" - data = dict(meta) - return app.backend.configs_update(config, data, get_gitactor(request)) - - -@app.delete("/configs/{config}/", tags=[Tags.congifs]) -async def config_delete(config: str): - """Delete a configuration""" - return app.backend.configs_delete(config) - - -# @ns_configs.route("//") -# class Config(Resource): -# # @ns_configs.marshal_with(m_config, skip_none=True) -# # def get(self, config): -# # "Retrieve a complete configuration" -# # return current_app.backend.configs_get(config) -# -# # @ns_configs.expect(m_config, validate=True) -# # def post(self, config): -# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" -# # data = request.json -# # return current_app.backend.configs_create(data, config, get_gitactor()) -# -# # @ns_configs.expect(m_meta, validate=True) -# # def put(self, config): -# # "Update an existing configuration" -# # data = request.json -# # return current_app.backend.configs_update(config, data, get_gitactor()) -# -# def delete(self, config): -# "Delete a configuration" -# return current_app.backend.configs_delete(config) - - -@app.post("/configs/{config}/clone/", tags=[Tags.congifs]) -async def config_clone_post(config: str, meta: Meta): - """Clone a configuration. New name is provided in POST data""" - data = dict(meta) - return app.backend.configs_clone(config, data) - - -# @ns_configs.route("//clone/") -# class ConfigClone(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config): -# "Clone a configuration. New name is provided in POST data" -# data = request.json -# return current_app.backend.configs_clone(config, data) -# - -@app.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta): - """Clone a configuration. New name is provided URL""" - data = dict(meta) - return app.backend.configs_clone(config, data, new_name) - - -# @ns_configs.route("//clone//") -# class ConfigCloneName(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config, new_name): -# "Clone a configuration. New name is provided URL" -# data = request.json -# return current_app.backend.configs_clone(config, data, new_name) - - -@app.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def config_list_version_get(config: str): - """Get all versions of a given configuration""" - return app.backend.configs_list_versions(config) - - -# @ns_configs.route("//v/") -# class ConfigListVersion(Resource): -# @ns_configs.marshal_with(m_version_log, skip_none=True) -# def get(self, config): -# "Get all versions of a given configuration" -# return current_app.backend.configs_list_versions(config) - - -@app.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) -async def config_version_get(config: str, version: str): - """Retrieve a specific version of a configuration""" - return app.backend.configs_get(config, version) - - -# @ns_configs.route("//v//") -# class ConfigVersion(Resource): -# def get(self, config, version): -# "Retrieve a specific version of a configuration" -# return current_app.backend.configs_get(config, version) - -@app.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) -async def config_revert_put(config: str, version: str, request: Request): - """Create a new version for a configuration from an old version""" - return app.backend.configs_revert(config, version, get_gitactor(request)) - - -# @ns_configs.route("//v//revert/") -# class ConfigRevert(Resource): -# def put(self, config, version): -# "Create a new version for a configuration from an old version" -# return current_app.backend.configs_revert(config, version, get_gitactor()) - - -############# -### Blobs ### -############# - - -@app.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) -async def blobs_resource_get(config: str): - """Retrieve the list of available blobs""" - res = app.backend.blobs_list(config) - return res - - -# @ns_configs.route("//b/") -# class BlobsResource(Resource): -# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of available blobs" -# res = current_app.backend.blobs_list(config) -# return res - -@app.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_resource_get(config: str, blob: str): - """Retrieve a blob""" - return app.backend.blobs_get(config, blob) - - -@app.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" - return app.backend.blobs_create( - config, blob, dict(blob_entry), get_gitactor(request) - ) - - -@app.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" - return app.backend.blobs_update( - config, blob, dict(blob_entry), get_gitactor(request) - ) - - -@app.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_get(config: str, blob: str, request: Request): - """Delete a blob""" - return app.backend.blobs_delete(config, blob, get_gitactor(request)) - - -# -# @ns_configs.route("//b//") -# class BlobResource(Resource): -# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) -# # def get(self, config, blob): -# # "Retrieve a blob" -# # return current_app.backend.blobs_get(config, blob) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def post(self, config, blob): -# # "Create a new blob" -# # return current_app.backend.blobs_create( -# # config, blob, request.json, get_gitactor() -# # ) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def put(self, config, blob): -# # "Replace a blob with new data" -# # return current_app.backend.blobs_update( -# # config, blob, request.json, get_gitactor() -# # ) -# -# def delete(self, config, blob): -# "Delete a blob" -# return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@app.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def blob_list_version_resource_get(config: str, blob: str): - "Retrieve the list of versions of a given blob" - res = app.backend.blobs_list_versions(config, blob) - return res - - -# @ns_configs.route("//b//v/") -# class BlobListVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob): -# "Retrieve the list of versions of a given blob" -# res = current_app.backend.blobs_list_versions(config, blob) -# return res - - -@app.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) -async def blob_version_resource_get(config: str, blob: str, version: str): - """Retrieve the given version of a blob""" - return app.backend.blobs_get(config, blob, version) - - -# @ns_configs.route("//b//v//") -# class BlobVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob, version): -# "Retrieve the given version of a blob" -# return current_app.backend.blobs_get(config, blob, version) - -@app.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) -async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): - """Create a new version for a blob from an old version""" - return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) - - -# -# @ns_configs.route("//b//v//revert/") -# class BlobRevertResource(Resource): -# def put(self, config, blob, version): -# "Create a new version for a blob from an old version" -# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) - - -################# -### DOCUMENTS ### -################# - -@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) -async def document_resource(config: str): - """Retrieve the list of existing documents in this configuration""" - res = app.backend.documents_list(config) - return res - - -# -# @ns_configs.route("//d/") -# class DocumentsResource(Resource): -# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of existing documents in this configuration" -# res = current_app.backend.documents_list(config) -# return res - - -@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) -async def document_resource_get(config: str, document: str): - """Get a complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - res = app.backend.documents_get(config, document) - res = {key: res[key] for key in list(models[document].__fields__.keys())} - return res - - -async def _filter(data, keys): - return {key: data[key] for key in keys} - - -@app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): - """Create a new complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) - if isValid is False: - raise HTTPException(500, "schema mismatched: \n" + err) - res = app.backend.documents_create( - config, document, data, get_gitactor(request) - ) - return res - - -@app.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): - """Update an existing document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) - if isValid is False: - raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = app.backend.documents_update( - config, document, data, get_gitactor(request) - ) - return res - - -@app.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_delete(config: str, document: str, request: Request): - """Delete/empty a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_delete(config, document, get_gitactor(request)) - return res - - -# @ns_configs.route("//d//") -# class DocumentResource(Resource): -# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) -# # def get(self, config, document): -# # "Get a complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_get(config, document) -# # return marshal(res, models[document], skip_none=True) -# # -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def post(self, config, document): -# # "Create a new complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched: \n" + err) -# # res = current_app.backend.documents_create( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def put(self, config, document): -# # "Update an existing document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) -# # res = current_app.backend.documents_update( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # def delete(self, config, document): -# # "Delete/empty a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_delete(config, document, get_gitactor()) -# # return res - - -@app.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str): - """Retrieve the existing versions of a given document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_list_versions(config, document) - res = {key: res[key] for key in list(VersionLog.__fields__.keys())} - return res - - -# -# @ns_configs.route("//d//v/") -# class DocumentListVersionResource(Resource): -# def get(self, config, document): -# "Retrieve the existing versions of a given document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_list_versions(config, document) -# return marshal(res, m_version_log, skip_none=True) - - -@app.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str): - """Get a given version of a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_get(config, document, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} - - -# @ns_configs.route("//d//v//") -# class DocumentVersionResource(Resource): -# def get(self, config, document, version): -# "Get a given version of a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_get(config, document, version) -# return marshal(res, models[document], skip_none=True) - -@app.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) -async def document_revert_resource_put(config: str, document: str, version: str, request: Request): - """Create a new version for a document from an old version""" - return app.backend.documents_revert( - config, document, version, get_gitactor(request) - ) - - -# @ns_configs.route("//d//v//revert/") -# class DocumentRevertResource(Resource): -# def put(self, config, document, version): -# "Create a new version for a document from an old version" -# return current_app.backend.documents_revert( -# config, document, version, get_gitactor() -# ) - - -############### -### ENTRIES ### -############### - -@app.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_get(config: str, document: str): - """Retrieve the list of entries in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_list(config, document) - return res # XXX: marshal - - -@app.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): - "Create an entry in a document" - if document not in models: - raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) - if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_create( - config, document, data, get_gitactor(request) - ) - return res - else: - raise HTTPException(500, "schema mismatched: \n" + err) - - -# @ns_configs.route("//d//e/") -# class EntriesResource(Resource): -# # def get(self, config, document): -# # "Retrieve the list of entries in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_list(config, document) -# # return res # XXX: marshal -# -# @ns_configs.expect(m_basic_entry, validate=True) -# def post(self, config, document): -# "Create an entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# isValid, err = validateJson(request.json, document) -# if isValid: -# data = marshal(request.json, models[document], skip_none=True) -# res = current_app.backend.entries_create( -# config, document, data, get_gitactor() -# ) -# return res -# else: -# abort(500, "schema mismatched: \n" + err) - - -@app.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_get(config: str, document: str, entry: str): - """Retrieve an entry from a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry) - return {key: res for key in list(models[document].__fields__.keys())} - - -@app.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): - """Update an entry in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) - if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - - res = app.backend.entries_update( - config, document, entry, data, get_gitactor(request) - ) - return res - else: - raise HTTPException(500, "schema mismatched: \n" + err) - - -@app.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): - """Delete an entry from a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_delete( - config, document, entry, get_gitactor(request) - ) - return res - - -# @ns_configs.route("//d//e//") -# class EntryResource(Resource): -# # def get(self, config, document, entry): -# # "Retrieve an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_get(config, document, entry) -# # return marshal(res, models[document], skip_none=True) -# -# # @ns_configs.expect(m_basic_entry, validate=True) -# # def put(self, config, document, entry): -# # "Update an entry in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # isValid, err = validateJson(request.json, document) -# # if isValid: -# # data = marshal(request.json, models[document], skip_none=True) -# # res = current_app.backend.entries_update( -# # config, document, entry, data, get_gitactor() -# # ) -# # return res -# # else: -# # abort(500, "schema mismatched: \n" + err) -# -# # def delete(self, config, document, entry): -# # "Delete an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_delete( -# # config, document, entry, get_gitactor() -# # ) -# # return res - - -@app.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str): - """Get the list of existing versions of a given entry in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_list_versions(config, document, entry) - return {key: res[key] for key in list(VersionLog.__fields__.keys())} - - -# -# @ns_configs.route("//d//e//v/") -# class EntryListVersionResource(Resource): -# def get(self, config, document, entry): -# "Get the list of existing versions of a given entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_list_versions(config, document, entry) -# return marshal(res, m_version_log, skip_none=True) - - -@app.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str): - """Get a given version of a document entry""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} - - -# @ns_configs.route( -# "//d//e//v//" -# ) -# class EntryVersionResource(Resource): -# def get(self, config, document, entry, version): -# "Get a given version of a document entry" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_get(config, document, entry, version) -# return marshal(res, models[document], skip_none=True) - - -################ -### Database ### -################ - -@app.get("/db/", tags=[Tags.db]) -async def db_resource_get(): - """Get the list of existing namespaces""" - return app.backend.ns_list() - - -# @ns_db.route("/") -# class DbResource(Resource): -# def get(self): -# "Get the list of existing namespaces" -# return current_app.backend.ns_list() - - -@app.get("/db/v/", tags=[Tags.db]) -async def db_query_resource_get(): - """List all existing versions of namespaces""" - return app.backend.ns_list_versions() - - -# -# @ns_db.route("/v/") -# class DbQueryResource(Resource): -# def get(self): -# "List all existing versions of namespaces" -# return current_app.backend.ns_list_versions() - - -@app.get("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_get(nsname: str): - """Get a complete namespace""" - try: - return app.backend.ns_get(nsname, version=None) - except KeyError: - raise HTTPException(404, "namespace [%s] does not exist" % nsname) - - -@app.post("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_post(nsname: str, db: DB, request: Request): - """Create a non-existing namespace from data""" - try: - return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) - except Exception: - raise HTTPException(409, "namespace [%s] already exists" % nsname) - - -@app.put("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_put(nsname: str, db: DB, request: Request): - """Merge data into a namespace""" - return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) - - -@app.delete("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_put(nsname: str, request: Request): - """Delete an existing namespace""" - try: - return app.backend.ns_delete(nsname, get_gitactor(request)) - except KeyError: - raise HTTPException(409, "namespace [%s] does not exist" % nsname) - - -# @ns_db.route("//") -# class NSResource(Resource): -# # def get(self, nsname): -# # "Get a complete namespace" -# # try: -# # return current_app.backend.ns_get(nsname, version=None) -# # except KeyError: -# # abort(404, "namespace [%s] does not exist" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def post(self, nsname): -# # "Create a non-existing namespace from data" -# # try: -# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) -# # except Exception: -# # abort(409, "namespace [%s] already exists" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def put(self, nsname): -# # "Merge data into a namespace" -# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) -# -# # def delete(self, nsname): -# # "Delete an existing namespace" -# # try: -# # return current_app.backend.ns_delete(nsname, get_gitactor()) -# # except KeyError: -# # abort(409, "namespace [%s] does not exist" % nsname) - - -@app.get("/db/{nsname}/v/{version}", tags=[Tags.db]) -async def ns_version_resource_get(nsname: str, version: str): - """Get a given version of a namespace""" - return app.backend.ns_get(nsname, version) - - -# @ns_db.route("//v//") -# class NSVersionResource(Resource): -# def get(self, nsname, version): -# "Get a given version of a namespace" -# return current_app.backend.ns_get(nsname, version) - - -@app.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) -async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): - """Create a new version for a namespace from an old version""" - try: - return app.backend.ns_revert(nsname, version, get_gitactor(request)) - except KeyError: - raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -# -# @ns_db.route("//v//revert/") -# class NSVersionResource(Resource): -# def put(self, nsname, version): -# "Create a new version for a namespace from an old version" -# try: -# return current_app.backend.ns_revert(nsname, version, get_gitactor()) -# except KeyError: -# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@app.post("/db/{nsname}/q/", tags=[Tags.db]) -async def ns_query_resource_post(nsname: str, request: Request): - """Run a JSON query on the namespace and returns the results""" - req_json = await request.json() - return app.backend.ns_query(nsname, req_json) - - -# @ns_db.route("//q/") -# class NSQueryResource(Resource): -# def post(self, nsname): -# "Run a JSON query on the namespace and returns the results" -# return current_app.backend.ns_query(nsname, request.json) -# - - -@app.get("/db/{nsname}/k/", tags=[Tags.db]) -async def keys_resource_get(nsname: str): - """List all keys of a given namespace""" - return app.backend.key_list(nsname) - - -# @ns_db.route("//k/") -# class KeysResource(Resource): -# def get(self, nsname): -# "List all keys of a given namespace" -# return current_app.backend.key_list(nsname) - -@app.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) -async def keys_list_versions_resource_get(nsname: str, key: str): - """Get all versions of a given key in namespace""" - return app.backend.key_list_versions(nsname, key) - - -# @ns_db.route("//k//v/") -# class KeysListVersionsResource(Resource): -# def get(self, nsname, key): -# "Get all versions of a given key in namespace" -# return current_app.backend.key_list_versions(nsname, key) -# - -@app.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_get(nsname: str, key: str): - """Retrieve a given key's value from a given namespace""" - return app.backend.key_get(nsname, key) - - -@app.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_put(nsname: str, key: str, request: Request): - """Create or update the value of a key""" - # check if "reblaze/k/" exists in system/schema-validation - req_json = await request.json() - - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(req_json, schema) - if isValid is False: - raise HTTPException(500, "schema mismatched") - return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) - - -@app.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_delete(nsname: str, key: str, request: Request): - """Delete a key""" - return app.backend.key_delete(nsname, key, get_gitactor(request)) - - -# @ns_db.route("//k//") -# class KeyResource(Resource): -# # def get(self, nsname, key): -# # "Retrieve a given key's value from a given namespace" -# # return current_app.backend.key_get(nsname, key) -# -# # def put(self, nsname, key): -# # "Create or update the value of a key" -# # # check if "reblaze/k/" exists in system/schema-validation -# # if nsname != "system": -# # keyName = nsname + "/k/" + key -# # schemas = current_app.backend.key_get("system", "schema-validation") -# # schema = None -# # # find schema if exists and validate the json input -# # for item in schemas.items(): -# # if item[0] == keyName: -# # schema = item[1] -# # break -# # if schema: -# # isValid = validateDbJson(request.json, schema) -# # if isValid is False: -# # abort(500, "schema mismatched") -# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) -# -# # def delete(self, nsname, key): -# # "Delete a key" -# # return current_app.backend.key_delete(nsname, key, get_gitactor()) - - -############# -### Tools ### -############# - - -req_fetch_parser = reqparse.RequestParser() -req_fetch_parser.add_argument("url", location="args", help="url to retrieve") - - -@app.get("/tools/fetch", tags=[Tags.tools]) -async def fetch_resource_get(url: str): - """Fetch an URL""" - try: - r = requests.get(url) - except Exception as e: - raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) - return r.content - - -# @ns_tools.route("/fetch") -# class FetchResource(Resource): -# @ns_tools.expect(req_fetch_parser, validate=True) -# def get(self): -# "Fetch an URL" -# args = req_fetch_parser.parse_args() -# try: -# r = requests.get(args.url) -# except Exception as e: -# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) -# return make_response(r.content) - - -@app.put("/tools/publish/{config}/", tags=[Tags.tools]) -@app.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) -async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): - """Push configuration to s3 buckets""" - conf = app.backend.configs_get(config, version) - ok = True - status = [] - req_json = await request.json() - if type(req_json) is not list: - raise HTTPException(400, "body must be a list") - for bucket in buckets: - logs = [] - try: - cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} - ) - return {"ok": ok, "status": status} - - -# @ns_tools.route("/publish//") -# @ns_tools.route("/publish//v//") -# class PublishResource(Resource): -# @ns_tools.expect([m_bucket], validate=True) -# def put(self, config, version=None): -# "Push configuration to s3 buckets" -# conf = current_app.backend.configs_get(config, version) -# ok = True -# status = [] -# if type(request.json) is not list: -# abort(400, "body must be a list") -# for bucket in request.json: -# logs = [] -# try: -# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) -# except Exception as e: -# ok = False -# s = False -# msg = repr(e) -# else: -# s = True -# msg = "ok" -# status.append( -# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} -# ) -# return make_response({"ok": ok, "status": status}) - - -@app.put("/tools/gitpush/", tags=[Tags.tools]) -async def git_push_resource_put(git_urls: List[GitUrl]): - """Push git configuration to remote git repositories""" - ok = True - status = [] - for giturl in git_urls: - try: - app.backend.gitpush(dict(giturl)["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) - return {"ok": ok, "status": status} - - -# @ns_tools.route("/gitpush/") -# class GitPushResource(Resource): -# @ns_tools.expect([m_giturl], validate=True) -# def put(self): -# "Push git configuration to remote git repositories" -# ok = True -# status = [] -# for giturl in request.json: -# try: -# current_app.backend.gitpush(giturl["giturl"]) -# except Exception as e: -# msg = repr(e) -# s = False -# else: -# msg = "ok" -# s = True -# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) -# return make_response({"ok": ok, "status": status}) - - -@app.put("/tools/gitfetch/", tags=[Tags.tools]) -async def git_fetch_resource_put(giturl: GitUrl): - """Fetch git configuration from specified remote repository""" - ok = True - try: - app.backend.gitfetch(dict(giturl)["giturl"]) - except Exception as e: - ok = False - msg = repr(e) - else: - msg = "ok" - return {"ok": ok, "status": msg} - - -# @ns_tools.route("/gitfetch/") -# class GitFetchResource(Resource): -# @ns_tools.expect(m_giturl, validate=True) -# def put(self): -# "Fetch git configuration from specified remote repository" -# ok = True -# try: -# current_app.backend.gitfetch(request.json["giturl"]) -# except Exception as e: -# ok = False -# msg = repr(e) -# else: -# msg = "ok" -# return make_response({"ok": ok, "status": msg}) - - -if __name__ == '__main__': - print("hi") From a006706b6dd29d1d2fe6ffa52dfbd1bfe24e2529 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Wed, 16 Nov 2022 14:55:16 +0200 Subject: [PATCH 23/55] initial commit - validation models defined --- .../curieconf/confserver/v3/fast_api.py | 752 ++++++++++++++++++ 1 file changed, 752 insertions(+) create mode 100644 curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py new file mode 100644 index 000000000..0dcff4755 --- /dev/null +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -0,0 +1,752 @@ +import datetime +import typing +from typing import Optional, List, Union + +from fastapi import FastAPI, Request +from pydantic import BaseModel, Field, validator +import random # needed for generating a random number for an API +import uvicorn # optional if you run it directly from terminal +import jsonschema + +# monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type +jsonschema.Draft4Validator = jsonschema.Draft3Validator + +from curieconf import utils +from curieconf.utils import cloud +from curieconf.confserver import app + +import requests +from jsonschema import validate +from pathlib import Path +import json + +# api_bp = Blueprint("api_v3", __name__) +# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") + +# ns_configs = api.namespace("configs", description="Configurations") +# ns_db = api.namespace("db", description="Database") +# ns_tools = api.namespace("tools", description="Tools") + + +############## +### MODELS ### +############## + + +### Models for documents +anyTypeUnion = Union[int, float, bool, object, list, None] +anyOp = Optional[object] +anyType = ["number", "string", "boolean", "object", "array", "null"] + + +# class AnyType(fields.Raw): +# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] + + +# limit + +class Threshold(BaseModel): + limit: int + action: str + + +# m_threshold = api.model( +# "Rate Limit Threshold", +# { +# "limit": fields.Integer(required=True), +# "action": fields.String(required=True), +# }, +# ) + +class Limit(BaseModel): + id: str + name: str + description: Optional[str] + _global: bool = Field(alias="global") + active: bool + timeframe: int + thresholds: List[Threshold] + include: typing.Any + exclude: typing.Any + key: anyTypeUnion + pairwith: typing.Any + tags: List[str] + + +# m_limit = api.model( +# "Rate Limit", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "global": fields.Boolean(required=True), +# "active": fields.Boolean(required=True), +# "timeframe": fields.Integer(required=True), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "include": fields.Raw(required=True), +# "exclude": fields.Raw(required=True), +# "key": AnyType(required=True), +# "pairwith": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# }, +# ) + +# securitypolicy +class SecProfileMap(BaseModel): + id: str + name: str + description: str + match: str + acl_profile: str + acl_active: bool + content_filter_profile: str + content_filter_active: bool + limit_ids: Optional[list] + + +# m_secprofilemap = api.model( +# "Security Profile Map", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.String(required=True), +# "acl_profile": fields.String(required=True), +# "acl_active": fields.Boolean(required=True), +# "content_filter_profile": fields.String(required=True), +# "content_filter_active": fields.Boolean(required=True), +# "limit_ids": fields.List(fields.Raw()), +# }, +# ) + +# TODO = deprecated? +# m_map = api.model( +# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} +# ) + +class SecurityPolicy(BaseModel): + id: str + name: str + description: str + tags: List[str] + match: str + session: anyTypeUnion + session_ids: anyTypeUnion + map: List[SecProfileMap] + + +# m_securitypolicy = api.model( +# "Security Policy", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String()), +# "match": fields.String(required=True), +# "session": AnyType(), +# "session_ids": AnyType(), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# }, +# ) + +# content filter rule + +class ContentFilterRule(BaseModel): + id: str + name: str + msg: str + operand: str + severity: int + certainity: int + category: str + subcategory: str + risk: int + tags: Optional[List[str]] + description: Optional[str] + + +# m_contentfilterrule = api.model( +# "Content Filter Rule", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "msg": fields.String(required=True), +# "operand": fields.String(required=True), +# "severity": fields.Integer(required=True), +# "certainity": fields.Integer(required=True), +# "category": fields.String(required=True), +# "subcategory": fields.String(required=True), +# "risk": fields.Integer(required=True), +# "tags": fields.List(fields.String()), +# "description": fields.String(), +# }, +# ) + +# content filter profile +class ContentFilterProfile(BaseModel): + id: str + name: str + description: Optional[str] + ignore_alphanum: bool + args: typing.Any + headers: typing.Any + cookies: typing.Any + path: typing.Any + allsections: typing.Any + decoding: typing.Any + masking_seed: str + content_type: Optional[List[str]] + active: Optional[List[str]] + report: Optional[List[str]] + ignore: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + ignore_body: bool + + +# m_contentfilterprofile = api.model( +# "Content Filter Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "ignore_alphanum": fields.Boolean(required=True), +# "args": fields.Raw(required=True), +# "headers": fields.Raw(required=True), +# "cookies": fields.Raw(required=True), +# "path": fields.Raw(required=True), +# "allsections": fields.Raw(), +# "decoding": fields.Raw(required=True), +# "masking_seed": fields.String(required=True), +# "content_type": fields.List(fields.String()), +# "active": fields.List(fields.String()), +# "report": fields.List(fields.String()), +# "ignore": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# "ignore_body": fields.Boolean(required=True), +# }, +# ) + +# aclprofile +class ACLProfile(BaseModel): + id: str + name: str + description: Optional[str] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + + +# m_aclprofile = api.model( +# "ACL Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# }, +# ) + +# Global Filter +class GlobalFilter(BaseModel): + id: str + name: str + source: str + mdate: str + description: str + active: bool + action: typing.Any + tags: Optional[List[str]] + rule: anyTypeUnion + + +# m_glbalfilter = api.model( +# "Global Filter", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "source": fields.String(required=True), +# "mdate": fields.String(required=True), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# "action": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# "rule": AnyType(), +# }, +# ) + +# Flow Control + +class FlowControl(BaseModel): + id: str + name: str + timeframe: int + key: List[typing.Any] + sequence: List[typing.Any] + tags: Optional[List[str]] + include: Optional[List[str]] + exclude: Optional[List[str]] + description: Optional[str] + active: bool + + +# +# m_flowcontrol = api.model( +# "Flow Control", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "timeframe": fields.Integer(required=True), +# "key": fields.List(fields.Raw(required=True)), +# "sequence": fields.List(fields.Raw(required=True)), +# "tags": fields.List(fields.String()), +# "include": fields.List(fields.String()), +# "exclude": fields.List(fields.String()), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# }, +# ) + +# Action + +class Action(BaseModel): + id: str + name: str + description: Optional[str] + tags: List[str] + params: typing.Any + type: str + + +# m_action = api.model( +# "Action", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String(required=True)), +# "params": fields.Raw(), +# "type": fields.String(required=True), +# }, +# ) + +# Virtual Tag +class VirtualTag(BaseModel): + id: str + name: str + description: Optional[str] + match: List[typing.Any] + + +# +# m_virtualtag = api.model( +# "Virtual Tag", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.List(fields.Raw(required=True)), +# }, +# ) + +# custom +class Custom(BaseModel): + id: str + name: str + + +# m_custom = api.model( +# "Custom", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +### mapping from doc name to model + +models = { + "ratelimits": Limit, + "securitypolicies": SecurityPolicy, + "contentfilterrules": ContentFilterRule, + "contentfilterprofiles": ContentFilterProfile, + "aclprofiles": ACLProfile, + "globalfilters": GlobalFilter, + "flowcontrol": FlowControl, + "actions": Action, + "virtualtags": Custom, + "custom": Custom, +} + + +### Other models +class DocumentMask(BaseModel): + id: str + name: str + description: str + map: Optional[List[SecProfileMap]] + include: Optional[List[typing.Any]] + exclude: Optional[List[typing.Any]] + tags: Optional[List[str]] + active: Optional[List[typing.Any]] + action: typing.Any + sequence: Optional[List[typing.Any]] + timeframe: Optional[int] + thresholds: Optional[List[Threshold]] + pairwith: typing.Any + content_type: Optional[List[str]] + params: typing.Any + decoding: typing.Any + category: Optional[str] + subcategory: Optional[str] + risk: Optional[int] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + match: Optional[str] = "j" + _type: Optional[str] = Field(alias="type") + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# m_document_mask = api.model( +# "Mask for document", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(required=True), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# "include": fields.Wildcard(fields.Raw()), +# "exclude": fields.Wildcard(fields.Raw()), +# "tags": fields.List(fields.String()), +# "active": fields.Wildcard(fields.Raw()), +# "action": fields.Raw(), +# "sequence": fields.List(fields.Raw()), +# "timeframe": fields.Integer(), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "pairwith": fields.Raw(), +# "content_type": fields.List(fields.String()), +# "params": fields.Raw(), +# "decoding": fields.Raw(), +# "category": fields.String(), +# "subcategory": fields.String(), +# "risk": fields.Integer(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "match": fields.String(), +# "type": fields.String(), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class VersionLog(BaseModel): + version: Optional[str] + # TODO - dt_format="iso8601" + date: Optional[datetime.datetime] + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# +# m_version_log = api.model( +# "Version log", +# { +# "version": fields.String(), +# "date": fields.DateTime(dt_format="iso8601"), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class Meta(BaseModel): + id: str + description: str + date: Optional[datetime.datetime] + logs: Optional[List[VersionLog]] = [] + version: Optional[str] + + +# m_meta = api.model( +# "Meta", +# { +# "id": fields.String(required=True), +# "description": fields.String(required=True), +# "date": fields.DateTime(), +# "logs": fields.List(fields.Nested(m_version_log), default=[]), +# "version": fields.String(), +# }, +# ) + +class BlobEntry(BaseModel): + format: str + blob: anyTypeUnion + + +# m_blob_entry = api.model( +# "Blob Entry", +# { +# "format": fields.String(required=True), +# "blob": AnyType(), +# }, +# ) + +class BlobListEntry(BaseModel): + name: Optional[str] + + +# m_blob_list_entry = api.model( +# "Blob ListEntry", +# { +# "name": fields.String(), +# }, +# ) + +class DocumentListEntry(BaseModel): + name: Optional[str] + entries: Optional[int] + + +# m_document_list_entry = api.model( +# "Document ListEntry", +# { +# "name": fields.String(), +# "entries": fields.Integer(), +# }, +# ) + +class ConfigDocuments(BaseModel): + ratelimits: Optional[List[models["ratelimits"]]] = [] + securitypolicies: Optional[List[models["securitypolicies"]]] = [] + contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] + contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] + aclprofiles: Optional[List[models["aclprofiles"]]] = [] + globalfilters: Optional[List[models["globalfilters"]]] = [] + flowcontrol: Optional[List[models["flowcontrol"]]] = [] + actions: Optional[List[models["actions"]]] = [] + virtualtags: Optional[List[models["virtualtags"]]] = [] + custom: Optional[List[models["custom"]]] = [] + + +# m_config_documents = api.model( +# "Config Documents", +# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, +# ) + + +class ConfigBlobs(BaseModel): + geolite2asn: Optional[List[Optional[BlobEntry]]] + geolite2country: Optional[List[Optional[BlobEntry]]] + geolite2city: Optional[List[Optional[BlobEntry]]] + customconf: Optional[List[Optional[BlobEntry]]] + + +# m_config_blobs = api.model( +# "Config Blobs", +# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, +# ) + +class ConfigDeleteBlobs(BaseModel): + geolite2asn: Optional[bool] + geolite2country: Optional[bool] + geolite2city: Optional[bool] + customconf: Optional[bool] + + +# m_config_delete_blobs = api.model( +# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} +# ) + +class Config(BaseModel): + meta: Meta = {} + documents: ConfigDocuments = {} + blobs: ConfigBlobs = {} + delete_documents: ConfigDocuments = {} + delete_blobs: ConfigDeleteBlobs = {} + + +# m_config = api.model( +# "Config", +# { +# "meta": fields.Nested(m_meta, default={}), +# "documents": fields.Nested(m_config_documents, default={}), +# "blobs": fields.Nested(m_config_blobs, default={}), +# "delete_documents": fields.Nested(m_config_documents, default={}), +# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), +# }, +# ) + +class Edit(BaseModel): + path: str + value: str + + +# m_edit = api.model( +# "Edit", +# { +# "path": fields.String(required=True), +# "value": fields.String(required=True), +# }, +# ) + +class BasicEntry(BaseModel): + id: str + name: str + description: Optional[str] + + +# m_basic_entry = api.model( +# "Basic Document Entry", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# }, +# ) + +### Publish + +class Bucket(BaseModel): + name: str + url: str + + +# m_bucket = api.model( +# "Bucket", +# { +# "name": fields.String(required=True), +# "url": fields.String(required=True), +# }, +# ) + +### Git push & pull + +class GitUrl(BaseModel): + giturl: str + + +# m_giturl = api.model( +# "GitUrl", +# { +# "giturl": fields.String(required=True), +# }, +# ) + +### Db +class DB(BaseModel): + pass + + +# m_db = api.model("db", {}) + + +### Document Schema validation + + +def validateJson(json_data, schema_type): + try: + validate(instance=json_data, schema=schema_type_map[schema_type]) + except jsonschema.exceptions.ValidationError as err: + print(str(err)) + return False, str(err) + return True, "" + + +### DB Schema validation + + +def validateDbJson(json_data, schema): + try: + validate(instance=json_data, schema=schema) + except jsonschema.exceptions.ValidationError as err: + print(str(err)) + return False + return True + + +### Set git actor according to config & defined HTTP headers + + +def get_gitactor(request): + email, username = "", "" + email_header = app.options.get("trusted_email_header", None) + if email_header: + email = request.headers.get(email_header, "") + username_header = app.options.get("trusted_username_header", None) + if username_header: + username = request.headers.get(username_header, "") + return app.backend.prepare_actor(username, email) + + +base_path = Path(__file__).parent +# base_path = "/etc/curiefense/json/" +acl_profile_file_path = (base_path / "./json/acl-profile.schema").resolve() +with open(acl_profile_file_path) as json_file: + acl_profile_schema = json.load(json_file) +ratelimits_file_path = (base_path / "./json/rate-limits.schema").resolve() +with open(ratelimits_file_path) as json_file: + ratelimits_schema = json.load(json_file) +securitypolicies_file_path = (base_path / "./json/security-policies.schema").resolve() +with open(securitypolicies_file_path) as json_file: + securitypolicies_schema = json.load(json_file) +content_filter_profile_file_path = ( + base_path / "./json/content-filter-profile.schema" +).resolve() +with open(content_filter_profile_file_path) as json_file: + content_filter_profile_schema = json.load(json_file) +globalfilters_file_path = (base_path / "./json/global-filters.schema").resolve() +with open(globalfilters_file_path) as json_file: + globalfilters_schema = json.load(json_file) +flowcontrol_file_path = (base_path / "./json/flow-control.schema").resolve() +with open(flowcontrol_file_path) as json_file: + flowcontrol_schema = json.load(json_file) +content_filter_rule_file_path = ( + base_path / "./json/content-filter-rule.schema" +).resolve() +with open(content_filter_rule_file_path) as json_file: + content_filter_rule_schema = json.load(json_file) +action_file_path = (base_path / "./json/action.schema").resolve() +with open(action_file_path) as json_file: + action_schema = json.load(json_file) +virtualtag_file_path = (base_path / "./json/virtual-tags.schema").resolve() +with open(virtualtag_file_path) as json_file: + virtual_tags_schema = json.load(json_file) +custom_file_path = (base_path / "./json/custom.schema").resolve() +with open(custom_file_path) as json_file: + custom_schema = json.load(json_file) +schema_type_map = { + "ratelimits": ratelimits_schema, + "securitypolicies": securitypolicies_schema, + "contentfilterprofiles": content_filter_profile_schema, + "aclprofiles": acl_profile_schema, + "globalfilters": globalfilters_schema, + "flowcontrol": flowcontrol_schema, + "contentfilterrules": content_filter_rule_schema, + "actions": action_schema, + "virtualtags": virtual_tags_schema, + "custom": custom_schema, +} + + + + + +if __name__ == '__main__': + print("hi") From 4a113f9dc48cb1b9f4b41e066785f112ad8c9b06 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Wed, 16 Nov 2022 21:00:59 +0200 Subject: [PATCH 24/55] end of day backup --- .../curieconf/confserver/v3/fast_api.py | 703 +++++++++++++++++- 1 file changed, 699 insertions(+), 4 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 0dcff4755..68f820566 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1,8 +1,9 @@ import datetime import typing +from enum import Enum from typing import Optional, List, Union -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, HTTPException from pydantic import BaseModel, Field, validator import random # needed for generating a random number for an API import uvicorn # optional if you run it directly from terminal @@ -11,9 +12,23 @@ # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type jsonschema.Draft4Validator = jsonschema.Draft3Validator -from curieconf import utils -from curieconf.utils import cloud -from curieconf.confserver import app +# from curieconf import utils +# from curieconf.utils import cloud +# from curieconf.confserver import app + + +# TODO: TEMP DEFINITIONS +import os + +app = FastAPI() +app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") +options = {} +val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) +if val: + options["trusted_username_header"] = val +val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) +if val: + options["trusted_email_header"] = val import requests from jsonschema import validate @@ -745,7 +760,687 @@ def get_gitactor(request): } +class Tags(Enum): + congifs = "configs" + db = "db" + tools = "tools" + + +################ +### CONFIGS ### +################ + +@app.get("/configs/", tags=[Tags.congifs], response_model=Meta) +async def configs_get(): + """Get the detailed list of existing configurations""" + return app.backend.configs_list() + + +@app.post("/configs/", tags=[Tags.congifs]) +async def configs_post(config: Config, request: Request): + """Create a new configuration""" + data = dict(config) + return app.backend.configs_create(data, get_gitactor(request)) + + +# +# @ns_configs.route("/") +# class Configs(Resource): +# # @ns_configs.marshal_list_with(m_meta, skip_none=True) +# # def get(self): +# # "Get the detailed list of existing configurations" +# # return current_app.backend.configs_list() +# +# @ns_configs.expect(m_config, validate=True) +# def post(self): +# "Create a new configuration" +# data = request.json +# return current_app.backend.configs_create(data, get_gitactor()) + +@app.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +async def config_get(config: str): + """Retrieve a complete configuration""" + return app.backend.configs_get(config) + + +@app.post("/configs/{config}/", tags=[Tags.congifs]) +async def config_post(config: str, m_config: Config, request: Request): + "Create a new configuration. Configuration name in URL overrides configuration in POST data" + data = dict(m_config) + return app.backend.configs_create(data, config, get_gitactor(request)) + + +@app.put("/configs/{config}/", tags=[Tags.congifs]) +async def config_put(config: str, meta: Meta, request: Request): + """Update an existing configuration""" + data = dict(meta) + return app.backend.configs_update(config, data, get_gitactor(request)) + + +@app.delete("/configs/{config}/", tags=[Tags.congifs]) +async def config_delete(config: str): + """Delete a configuration""" + return app.backend.configs_delete(config) + + +# @ns_configs.route("//") +# class Config(Resource): +# # @ns_configs.marshal_with(m_config, skip_none=True) +# # def get(self, config): +# # "Retrieve a complete configuration" +# # return current_app.backend.configs_get(config) +# +# # @ns_configs.expect(m_config, validate=True) +# # def post(self, config): +# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" +# # data = request.json +# # return current_app.backend.configs_create(data, config, get_gitactor()) +# +# # @ns_configs.expect(m_meta, validate=True) +# # def put(self, config): +# # "Update an existing configuration" +# # data = request.json +# # return current_app.backend.configs_update(config, data, get_gitactor()) +# +# def delete(self, config): +# "Delete a configuration" +# return current_app.backend.configs_delete(config) + + +@app.post("/configs/{config}/clone/", tags=[Tags.congifs]) +async def config_clone_post(config: str, meta: Meta): + """Clone a configuration. New name is provided in POST data""" + data = dict(meta) + return app.backend.configs_clone(config, data) + + +# @ns_configs.route("//clone/") +# class ConfigClone(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config): +# "Clone a configuration. New name is provided in POST data" +# data = request.json +# return current_app.backend.configs_clone(config, data) +# + +@app.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) +async def config_clone_name_post(config: str, new_name: str, meta: Meta): + """Clone a configuration. New name is provided URL""" + data = dict(meta) + return app.backend.configs_clone(config, data, new_name) + + +# @ns_configs.route("//clone//") +# class ConfigCloneName(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config, new_name): +# "Clone a configuration. New name is provided URL" +# data = request.json +# return current_app.backend.configs_clone(config, data, new_name) + + +@app.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def config_list_version_get(config: str): + """Get all versions of a given configuration""" + return app.backend.configs_list_versions(config) + + +# @ns_configs.route("//v/") +# class ConfigListVersion(Resource): +# @ns_configs.marshal_with(m_version_log, skip_none=True) +# def get(self, config): +# "Get all versions of a given configuration" +# return current_app.backend.configs_list_versions(config) + + +@app.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +async def config_version_get(config: str, version: str): + """Retrieve a specific version of a configuration""" + return app.backend.configs_get(config, version) + + +# @ns_configs.route("//v//") +# class ConfigVersion(Resource): +# def get(self, config, version): +# "Retrieve a specific version of a configuration" +# return current_app.backend.configs_get(config, version) + +@app.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +async def config_revert_put(config: str, version: str, request: Request): + """Create a new version for a configuration from an old version""" + return app.backend.configs_revert(config, version, get_gitactor(request)) + + +# @ns_configs.route("//v//revert/") +# class ConfigRevert(Resource): +# def put(self, config, version): +# "Create a new version for a configuration from an old version" +# return current_app.backend.configs_revert(config, version, get_gitactor()) + + +############# +### Blobs ### +############# + + +@app.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +async def blobs_resource_get(config: str): + """Retrieve the list of available blobs""" + res = app.backend.blobs_list(config) + return res + + +# @ns_configs.route("//b/") +# class BlobsResource(Resource): +# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of available blobs" +# res = current_app.backend.blobs_list(config) +# return res + +@app.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +async def blob_resource_get(config: str, blob: str): + """Retrieve a blob""" + return app.backend.blobs_get(config, blob) + + +@app.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_create( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@app.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_update( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@app.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_get(config: str, blob: str, request: Request): + """Delete a blob""" + return app.backend.blobs_delete(config, blob, get_gitactor(request)) + + +# +# @ns_configs.route("//b//") +# class BlobResource(Resource): +# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) +# # def get(self, config, blob): +# # "Retrieve a blob" +# # return current_app.backend.blobs_get(config, blob) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def post(self, config, blob): +# # "Create a new blob" +# # return current_app.backend.blobs_create( +# # config, blob, request.json, get_gitactor() +# # ) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def put(self, config, blob): +# # "Replace a blob with new data" +# # return current_app.backend.blobs_update( +# # config, blob, request.json, get_gitactor() +# # ) +# +# def delete(self, config, blob): +# "Delete a blob" +# return current_app.backend.blobs_delete(config, blob, get_gitactor()) + + +@app.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def blob_list_version_resource_get(config: str, blob: str): + "Retrieve the list of versions of a given blob" + res = app.backend.blobs_list_versions(config, blob) + return res + + +# @ns_configs.route("//b//v/") +# class BlobListVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob): +# "Retrieve the list of versions of a given blob" +# res = current_app.backend.blobs_list_versions(config, blob) +# return res + + +@app.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +async def blob_version_resource_get(config: str, blob: str, version: str): + """Retrieve the given version of a blob""" + return app.backend.blobs_get(config, blob, version) + + +# @ns_configs.route("//b//v//") +# class BlobVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob, version): +# "Retrieve the given version of a blob" +# return current_app.backend.blobs_get(config, blob, version) + +@app.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): + """Create a new version for a blob from an old version""" + return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + +# +# @ns_configs.route("//b//v//revert/") +# class BlobRevertResource(Resource): +# def put(self, config, blob, version): +# "Create a new version for a blob from an old version" +# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) + + +################# +### DOCUMENTS ### +################# + +@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model= DocumentListEntry) +async def document_resource(config:str): + """Retrieve the list of existing documents in this configuration""" + res = app.backend.documents_list(config) + return res + +# +# @ns_configs.route("//d/") +# class DocumentsResource(Resource): +# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of existing documents in this configuration" +# res = current_app.backend.documents_list(config) +# return res + + +@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model= DocumentMask ) +async def get(config: str, document: str): + pass +@app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def post(config: str, document: str, basic_entry: List[BasicEntry]): + "Create a new complete document" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = dict[basic_entry] + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched: \n" + err) + res = current_app.backend.documents_create( + config, document, data, get_gitactor() + ) + return res + + + +@ns_configs.route("//d//") +class DocumentResource(Resource): + @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) + def get(self, config, document): + "Get a complete document" + if document not in models: + + abort(404, "document does not exist") + res = current_app.backend.documents_get(config, document) + return marshal(res, models[document], skip_none=True) + + @ns_configs.expect([m_basic_entry], validate=True) + def post(self, config, document): + "Create a new complete document" + if document not in models: + abort(404, "document does not exist") + data = marshal(request.json, models[document], skip_none=True) + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched: \n" + err) + res = current_app.backend.documents_create( + config, document, data, get_gitactor() + ) + return res + + @ns_configs.expect([m_basic_entry], validate=True) + def put(self, config, document): + "Update an existing document" + if document not in models: + abort(404, "document does not exist") + data = marshal(request.json, models[document], skip_none=True) + for entry in request.json: + isValid, err = validateJson(entry, document) + if isValid is False: + abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = current_app.backend.documents_update( + config, document, data, get_gitactor() + ) + return res + + def delete(self, config, document): + "Delete/empty a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_delete(config, document, get_gitactor()) + return res + + +@ns_configs.route("//d//v/") +class DocumentListVersionResource(Resource): + def get(self, config, document): + "Retrieve the existing versions of a given document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_list_versions(config, document) + return marshal(res, m_version_log, skip_none=True) + + +@ns_configs.route("//d//v//") +class DocumentVersionResource(Resource): + def get(self, config, document, version): + "Get a given version of a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.documents_get(config, document, version) + return marshal(res, models[document], skip_none=True) + + +@ns_configs.route("//d//v//revert/") +class DocumentRevertResource(Resource): + def put(self, config, document, version): + "Create a new version for a document from an old version" + return current_app.backend.documents_revert( + config, document, version, get_gitactor() + ) + + +############### +### ENTRIES ### +############### + + +@ns_configs.route("//d//e/") +class EntriesResource(Resource): + def get(self, config, document): + "Retrieve the list of entries in a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_list(config, document) + return res # XXX: marshal + + @ns_configs.expect(m_basic_entry, validate=True) + def post(self, config, document): + "Create an entry in a document" + if document not in models: + abort(404, "document does not exist") + isValid, err = validateJson(request.json, document) + if isValid: + data = marshal(request.json, models[document], skip_none=True) + res = current_app.backend.entries_create( + config, document, data, get_gitactor() + ) + return res + else: + abort(500, "schema mismatched: \n" + err) + + +@ns_configs.route("//d//e//") +class EntryResource(Resource): + def get(self, config, document, entry): + "Retrieve an entry from a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_get(config, document, entry) + return marshal(res, models[document], skip_none=True) + + @ns_configs.expect(m_basic_entry, validate=True) + def put(self, config, document, entry): + "Update an entry in a document" + if document not in models: + abort(404, "document does not exist") + isValid, err = validateJson(request.json, document) + if isValid: + data = marshal(request.json, models[document], skip_none=True) + res = current_app.backend.entries_update( + config, document, entry, data, get_gitactor() + ) + return res + else: + abort(500, "schema mismatched: \n" + err) + + def delete(self, config, document, entry): + "Delete an entry from a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_delete( + config, document, entry, get_gitactor() + ) + return res + + +@ns_configs.route("//d//e//v/") +class EntryListVersionResource(Resource): + def get(self, config, document, entry): + "Get the list of existing versions of a given entry in a document" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_list_versions(config, document, entry) + return marshal(res, m_version_log, skip_none=True) + + +@ns_configs.route( + "//d//e//v//" +) +class EntryVersionResource(Resource): + def get(self, config, document, entry, version): + "Get a given version of a document entry" + if document not in models: + abort(404, "document does not exist") + res = current_app.backend.entries_get(config, document, entry, version) + return marshal(res, models[document], skip_none=True) + + +################ +### Database ### +################ + + +@ns_db.route("/") +class DbResource(Resource): + def get(self): + "Get the list of existing namespaces" + return current_app.backend.ns_list() + + +@ns_db.route("/v/") +class DbQueryResource(Resource): + def get(self): + "List all existing versions of namespaces" + return current_app.backend.ns_list_versions() + + +@ns_db.route("//") +class NSResource(Resource): + def get(self, nsname): + "Get a complete namespace" + try: + return current_app.backend.ns_get(nsname, version=None) + except KeyError: + abort(404, "namespace [%s] does not exist" % nsname) + + @ns_db.expect(m_db, validate=True) + def post(self, nsname): + "Create a non-existing namespace from data" + try: + return current_app.backend.ns_create(nsname, request.json, get_gitactor()) + except Exception: + abort(409, "namespace [%s] already exists" % nsname) + + @ns_db.expect(m_db, validate=True) + def put(self, nsname): + "Merge data into a namespace" + return current_app.backend.ns_update(nsname, request.json, get_gitactor()) + + def delete(self, nsname): + "Delete an existing namespace" + try: + return current_app.backend.ns_delete(nsname, get_gitactor()) + except KeyError: + abort(409, "namespace [%s] does not exist" % nsname) + + +@ns_db.route("//v//") +class NSVersionResource(Resource): + def get(self, nsname, version): + "Get a given version of a namespace" + return current_app.backend.ns_get(nsname, version) + + +@ns_db.route("//v//revert/") +class NSVersionResource(Resource): + def put(self, nsname, version): + "Create a new version for a namespace from an old version" + try: + return current_app.backend.ns_revert(nsname, version, get_gitactor()) + except KeyError: + abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@ns_db.route("//q/") +class NSQueryResource(Resource): + def post(self, nsname): + "Run a JSON query on the namespace and returns the results" + return current_app.backend.ns_query(nsname, request.json) + + +@ns_db.route("//k/") +class KeysResource(Resource): + def get(self, nsname): + "List all keys of a given namespace" + return current_app.backend.key_list(nsname) + + +@ns_db.route("//k//v/") +class KeysListVersionsResource(Resource): + def get(self, nsname, key): + "Get all versions of a given key in namespace" + return current_app.backend.key_list_versions(nsname, key) + + +@ns_db.route("//k//") +class KeyResource(Resource): + def get(self, nsname, key): + "Retrieve a given key's value from a given namespace" + return current_app.backend.key_get(nsname, key) + + def put(self, nsname, key): + "Create or update the value of a key" + # check if "reblaze/k/" exists in system/schema-validation + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = current_app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(request.json, schema) + if isValid is False: + abort(500, "schema mismatched") + return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) + + def delete(self, nsname, key): + "Delete a key" + return current_app.backend.key_delete(nsname, key, get_gitactor()) + + +############# +### Tools ### +############# + + +req_fetch_parser = reqparse.RequestParser() +req_fetch_parser.add_argument("url", location="args", help="url to retrieve") + + +@ns_tools.route("/fetch") +class FetchResource(Resource): + @ns_tools.expect(req_fetch_parser, validate=True) + def get(self): + "Fetch an URL" + args = req_fetch_parser.parse_args() + try: + r = requests.get(args.url) + except Exception as e: + abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) + return make_response(r.content) + + +@ns_tools.route("/publish//") +@ns_tools.route("/publish//v//") +class PublishResource(Resource): + @ns_tools.expect([m_bucket], validate=True) + def put(self, config, version=None): + "Push configuration to s3 buckets" + conf = current_app.backend.configs_get(config, version) + ok = True + status = [] + if type(request.json) is not list: + abort(400, "body must be a list") + for bucket in request.json: + logs = [] + try: + cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) + except Exception as e: + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} + ) + return make_response({"ok": ok, "status": status}) + + +@ns_tools.route("/gitpush/") +class GitPushResource(Resource): + @ns_tools.expect([m_giturl], validate=True) + def put(self): + "Push git configuration to remote git repositories" + ok = True + status = [] + for giturl in request.json: + try: + current_app.backend.gitpush(giturl["giturl"]) + except Exception as e: + msg = repr(e) + s = False + else: + msg = "ok" + s = True + status.append({"url": giturl["giturl"], "ok": s, "message": msg}) + return make_response({"ok": ok, "status": status}) + + +@ns_tools.route("/gitfetch/") +class GitFetchResource(Resource): + @ns_tools.expect(m_giturl, validate=True) + def put(self): + "Fetch git configuration from specified remote repository" + ok = True + try: + current_app.backend.gitfetch(request.json["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return make_response({"ok": ok, "status": msg}) if __name__ == '__main__': From 74cf3de403f8c8f1f45d0d1118e0e20e199c65ba Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 12:05:23 +0200 Subject: [PATCH 25/55] backup --- .../curieconf/confserver/v3/fast_api.py | 222 +++++++++++------- 1 file changed, 142 insertions(+), 80 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 68f820566..1396ff2c8 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1027,6 +1027,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request """Create a new version for a blob from an old version""" return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + # # @ns_configs.route("//b//v//revert/") # class BlobRevertResource(Resource): @@ -1039,12 +1040,13 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ### DOCUMENTS ### ################# -@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model= DocumentListEntry) -async def document_resource(config:str): +@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +async def document_resource(config: str): """Retrieve the list of existing documents in this configuration""" res = app.backend.documents_list(config) return res + # # @ns_configs.route("//d/") # class DocumentsResource(Resource): @@ -1055,104 +1057,164 @@ async def document_resource(config:str): # return res -@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model= DocumentMask ) -async def get(config: str, document: str): - pass +@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +async def document_resource_get(config: str, document: str): + """Get a complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + res = app.backend.documents_get(config, document) + res = {key: res[key] for key in list(models[document].__fields__.keys())} + return res + + +async def _filter(data, keys): + return {key: data[key] for key in keys} + @app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def post(config: str, document: str, basic_entry: List[BasicEntry]): - "Create a new complete document" +async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Create a new complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - data = dict[basic_entry] - for entry in request.json: - isValid, err = validateJson(entry, document) + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() + raise HTTPException(500, "schema mismatched: \n" + err) + res = app.backend.documents_create( + config, document, data, get_gitactor(request) ) return res +@app.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Update an existing document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") -@ns_configs.route("//d//") -class DocumentResource(Resource): - @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) - def get(self, config, document): - "Get a complete document" - if document not in models: - - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document) - return marshal(res, models[document], skip_none=True) + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = app.backend.documents_update( + config, document, data, get_gitactor(request) + ) + return res - @ns_configs.expect([m_basic_entry], validate=True) - def post(self, config, document): - "Create a new complete document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() - ) - return res - @ns_configs.expect([m_basic_entry], validate=True) - def put(self, config, document): - "Update an existing document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = current_app.backend.documents_update( - config, document, data, get_gitactor() - ) - return res +@app.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_delete(config: str, document: str, request: Request): + """Delete/empty a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_delete(config, document, get_gitactor(request)) + return res - def delete(self, config, document): - "Delete/empty a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_delete(config, document, get_gitactor()) - return res +# @ns_configs.route("//d//") +# class DocumentResource(Resource): +# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) +# # def get(self, config, document): +# # "Get a complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_get(config, document) +# # return marshal(res, models[document], skip_none=True) +# # +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def post(self, config, document): +# # "Create a new complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched: \n" + err) +# # res = current_app.backend.documents_create( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def put(self, config, document): +# # "Update an existing document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) +# # res = current_app.backend.documents_update( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # def delete(self, config, document): +# # "Delete/empty a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_delete(config, document, get_gitactor()) +# # return res -@ns_configs.route("//d//v/") -class DocumentListVersionResource(Resource): - def get(self, config, document): - "Retrieve the existing versions of a given document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_list_versions(config, document) - return marshal(res, m_version_log, skip_none=True) +@app.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) +async def document_list_version_resource_get(config: str, document: str): + """Retrieve the existing versions of a given document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_list_versions(config, document) + res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res -@ns_configs.route("//d//v//") -class DocumentVersionResource(Resource): - def get(self, config, document, version): - "Get a given version of a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document, version) - return marshal(res, models[document], skip_none=True) +# +# @ns_configs.route("//d//v/") +# class DocumentListVersionResource(Resource): +# def get(self, config, document): +# "Retrieve the existing versions of a given document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_list_versions(config, document) +# return marshal(res, m_version_log, skip_none=True) + + +@app.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) +async def document_version_resource_get(config: str, document: str, version: str): + """Get a given version of a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_get(config, document, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route("//d//v//") +# class DocumentVersionResource(Resource): +# def get(self, config, document, version): +# "Get a given version of a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_get(config, document, version) +# return marshal(res, models[document], skip_none=True) + +@app.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) +async def document_revert_resource_put(config: str, document: str, version: str, request: Request): + """Create a new version for a document from an old version""" + return app.backend.documents_revert( + config, document, version, get_gitactor(request) + ) -@ns_configs.route("//d//v//revert/") -class DocumentRevertResource(Resource): - def put(self, config, document, version): - "Create a new version for a document from an old version" - return current_app.backend.documents_revert( - config, document, version, get_gitactor() - ) +# @ns_configs.route("//d//v//revert/") +# class DocumentRevertResource(Resource): +# def put(self, config, document, version): +# "Create a new version for a document from an old version" +# return current_app.backend.documents_revert( +# config, document, version, get_gitactor() +# ) ############### From 09710fe690830f51df4aa9fd4c7f1f74563fa8df Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 13:50:16 +0200 Subject: [PATCH 26/55] configs done --- .../curieconf/confserver/v3/fast_api.py | 231 ++++++++++++------ 1 file changed, 156 insertions(+), 75 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index 1396ff2c8..e22aee840 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1208,6 +1208,7 @@ async def document_revert_resource_put(config: str, document: str, version: str, config, document, version, get_gitactor(request) ) + # @ns_configs.route("//d//v//revert/") # class DocumentRevertResource(Resource): # def put(self, config, document, version): @@ -1221,86 +1222,166 @@ async def document_revert_resource_put(config: str, document: str, version: str, ### ENTRIES ### ############### +@app.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_get(config: str, document: str): + """Retrieve the list of entries in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list(config, document) + return res # XXX: marshal -@ns_configs.route("//d//e/") -class EntriesResource(Resource): - def get(self, config, document): - "Retrieve the list of entries in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list(config, document) - return res # XXX: marshal - - @ns_configs.expect(m_basic_entry, validate=True) - def post(self, config, document): - "Create an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_create( - config, document, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - -@ns_configs.route("//d//e//") -class EntryResource(Resource): - def get(self, config, document, entry): - "Retrieve an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect(m_basic_entry, validate=True) - def put(self, config, document, entry): - "Update an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_update( - config, document, entry, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - def delete(self, config, document, entry): - "Delete an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_delete( - config, document, entry, get_gitactor() + +@app.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): + "Create an entry in a document" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + res = app.backend.entries_create( + config, document, data, get_gitactor(request) ) return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +# @ns_configs.route("//d//e/") +# class EntriesResource(Resource): +# # def get(self, config, document): +# # "Retrieve the list of entries in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_list(config, document) +# # return res # XXX: marshal +# +# @ns_configs.expect(m_basic_entry, validate=True) +# def post(self, config, document): +# "Create an entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# isValid, err = validateJson(request.json, document) +# if isValid: +# data = marshal(request.json, models[document], skip_none=True) +# res = current_app.backend.entries_create( +# config, document, data, get_gitactor() +# ) +# return res +# else: +# abort(500, "schema mismatched: \n" + err) + + +@app.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_get(config: str, document: str, entry: str): + """Retrieve an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry) + return {key: res for key in list(models[document].__fields__.keys())} + + +@app.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): + """Update an entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + + res = app.backend.entries_update( + config, document, entry, data, get_gitactor(request) + ) + return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +@app.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): + """Delete an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_delete( + config, document, entry, get_gitactor(request) + ) + return res -@ns_configs.route("//d//e//v/") -class EntryListVersionResource(Resource): - def get(self, config, document, entry): - "Get the list of existing versions of a given entry in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list_versions(config, document, entry) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route( - "//d//e//v//" -) -class EntryVersionResource(Resource): - def get(self, config, document, entry, version): - "Get a given version of a document entry" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry, version) - return marshal(res, models[document], skip_none=True) +# @ns_configs.route("//d//e//") +# class EntryResource(Resource): +# # def get(self, config, document, entry): +# # "Retrieve an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_get(config, document, entry) +# # return marshal(res, models[document], skip_none=True) +# +# # @ns_configs.expect(m_basic_entry, validate=True) +# # def put(self, config, document, entry): +# # "Update an entry in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # isValid, err = validateJson(request.json, document) +# # if isValid: +# # data = marshal(request.json, models[document], skip_none=True) +# # res = current_app.backend.entries_update( +# # config, document, entry, data, get_gitactor() +# # ) +# # return res +# # else: +# # abort(500, "schema mismatched: \n" + err) +# +# # def delete(self, config, document, entry): +# # "Delete an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_delete( +# # config, document, entry, get_gitactor() +# # ) +# # return res + + +@app.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) +async def entry_list_version_resource_get(config: str, document: str, entry: str): + """Get the list of existing versions of a given entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list_versions(config, document, entry) + return {key: res[key] for key in list(VersionLog.__fields__.keys())} + + +# +# @ns_configs.route("//d//e//v/") +# class EntryListVersionResource(Resource): +# def get(self, config, document, entry): +# "Get the list of existing versions of a given entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_list_versions(config, document, entry) +# return marshal(res, m_version_log, skip_none=True) + + +@app.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) +async def entry_version_resource_get(config: str, document: str, entry: str, version: str): + """Get a given version of a document entry""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route( +# "//d//e//v//" +# ) +# class EntryVersionResource(Resource): +# def get(self, config, document, entry, version): +# "Get a given version of a document entry" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_get(config, document, entry, version) +# return marshal(res, models[document], skip_none=True) ################ From fe0a633fd8a4c80d236fc90b5bafacf052eb29f3 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 15:30:54 +0200 Subject: [PATCH 27/55] db done --- .../curieconf/confserver/v3/fast_api.py | 310 ++++++++++++------ 1 file changed, 212 insertions(+), 98 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index e22aee840..e19c0c6ef 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -1388,116 +1388,230 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver ### Database ### ################ +@app.get("/db/", tags=[Tags.db]) +async def db_resource_get(): + """Get the list of existing namespaces""" + return app.backend.ns_list() -@ns_db.route("/") -class DbResource(Resource): - def get(self): - "Get the list of existing namespaces" - return current_app.backend.ns_list() +# @ns_db.route("/") +# class DbResource(Resource): +# def get(self): +# "Get the list of existing namespaces" +# return current_app.backend.ns_list() -@ns_db.route("/v/") -class DbQueryResource(Resource): - def get(self): - "List all existing versions of namespaces" - return current_app.backend.ns_list_versions() +@app.get("/db/v/", tags=[Tags.db]) +async def db_query_resource_get(): + """List all existing versions of namespaces""" + return app.backend.ns_list_versions() -@ns_db.route("//") -class NSResource(Resource): - def get(self, nsname): - "Get a complete namespace" - try: - return current_app.backend.ns_get(nsname, version=None) - except KeyError: - abort(404, "namespace [%s] does not exist" % nsname) - @ns_db.expect(m_db, validate=True) - def post(self, nsname): - "Create a non-existing namespace from data" - try: - return current_app.backend.ns_create(nsname, request.json, get_gitactor()) - except Exception: - abort(409, "namespace [%s] already exists" % nsname) +# +# @ns_db.route("/v/") +# class DbQueryResource(Resource): +# def get(self): +# "List all existing versions of namespaces" +# return current_app.backend.ns_list_versions() - @ns_db.expect(m_db, validate=True) - def put(self, nsname): - "Merge data into a namespace" - return current_app.backend.ns_update(nsname, request.json, get_gitactor()) - def delete(self, nsname): - "Delete an existing namespace" - try: - return current_app.backend.ns_delete(nsname, get_gitactor()) - except KeyError: - abort(409, "namespace [%s] does not exist" % nsname) +@app.get("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_get(nsname: str): + """Get a complete namespace""" + try: + return app.backend.ns_get(nsname, version=None) + except KeyError: + raise HTTPException(404, "namespace [%s] does not exist" % nsname) -@ns_db.route("//v//") -class NSVersionResource(Resource): - def get(self, nsname, version): - "Get a given version of a namespace" - return current_app.backend.ns_get(nsname, version) +@app.post("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_post(nsname: str, db: DB, request: Request): + """Create a non-existing namespace from data""" + try: + return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + except Exception: + raise HTTPException(409, "namespace [%s] already exists" % nsname) -@ns_db.route("//v//revert/") -class NSVersionResource(Resource): - def put(self, nsname, version): - "Create a new version for a namespace from an old version" - try: - return current_app.backend.ns_revert(nsname, version, get_gitactor()) - except KeyError: - abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@ns_db.route("//q/") -class NSQueryResource(Resource): - def post(self, nsname): - "Run a JSON query on the namespace and returns the results" - return current_app.backend.ns_query(nsname, request.json) - - -@ns_db.route("//k/") -class KeysResource(Resource): - def get(self, nsname): - "List all keys of a given namespace" - return current_app.backend.key_list(nsname) - - -@ns_db.route("//k//v/") -class KeysListVersionsResource(Resource): - def get(self, nsname, key): - "Get all versions of a given key in namespace" - return current_app.backend.key_list_versions(nsname, key) - - -@ns_db.route("//k//") -class KeyResource(Resource): - def get(self, nsname, key): - "Retrieve a given key's value from a given namespace" - return current_app.backend.key_get(nsname, key) - - def put(self, nsname, key): - "Create or update the value of a key" - # check if "reblaze/k/" exists in system/schema-validation - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = current_app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(request.json, schema) - if isValid is False: - abort(500, "schema mismatched") - return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) - - def delete(self, nsname, key): - "Delete a key" - return current_app.backend.key_delete(nsname, key, get_gitactor()) +@app.put("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, db: DB, request: Request): + """Merge data into a namespace""" + return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + + +@app.delete("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, request: Request): + """Delete an existing namespace""" + try: + return app.backend.ns_delete(nsname, get_gitactor(request)) + except KeyError: + raise HTTPException(409, "namespace [%s] does not exist" % nsname) + + +# @ns_db.route("//") +# class NSResource(Resource): +# # def get(self, nsname): +# # "Get a complete namespace" +# # try: +# # return current_app.backend.ns_get(nsname, version=None) +# # except KeyError: +# # abort(404, "namespace [%s] does not exist" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def post(self, nsname): +# # "Create a non-existing namespace from data" +# # try: +# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) +# # except Exception: +# # abort(409, "namespace [%s] already exists" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def put(self, nsname): +# # "Merge data into a namespace" +# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) +# +# # def delete(self, nsname): +# # "Delete an existing namespace" +# # try: +# # return current_app.backend.ns_delete(nsname, get_gitactor()) +# # except KeyError: +# # abort(409, "namespace [%s] does not exist" % nsname) + + +@app.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +async def ns_version_resource_get(nsname: str, version: str): + """Get a given version of a namespace""" + return app.backend.ns_get(nsname, version) + + +# @ns_db.route("//v//") +# class NSVersionResource(Resource): +# def get(self, nsname, version): +# "Get a given version of a namespace" +# return current_app.backend.ns_get(nsname, version) + + +@app.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) +async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): + """Create a new version for a namespace from an old version""" + try: + return app.backend.ns_revert(nsname, version, get_gitactor(request)) + except KeyError: + raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +# +# @ns_db.route("//v//revert/") +# class NSVersionResource(Resource): +# def put(self, nsname, version): +# "Create a new version for a namespace from an old version" +# try: +# return current_app.backend.ns_revert(nsname, version, get_gitactor()) +# except KeyError: +# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@app.post("/db/{nsname}/q/", tags=[Tags.db]) +async def ns_query_resource_post(nsname: str, request: Request): + """Run a JSON query on the namespace and returns the results""" + req_json = await request.json() + return app.backend.ns_query(nsname, req_json) + + +# @ns_db.route("//q/") +# class NSQueryResource(Resource): +# def post(self, nsname): +# "Run a JSON query on the namespace and returns the results" +# return current_app.backend.ns_query(nsname, request.json) +# + + +@app.get("/db/{nsname}/k/", tags=[Tags.db]) +async def keys_resource_get(nsname: str): + """List all keys of a given namespace""" + return app.backend.key_list(nsname) + + +# @ns_db.route("//k/") +# class KeysResource(Resource): +# def get(self, nsname): +# "List all keys of a given namespace" +# return current_app.backend.key_list(nsname) + +@app.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) +async def keys_list_versions_resource_get(nsname: str, key: str): + """Get all versions of a given key in namespace""" + return app.backend.key_list_versions(nsname, key) + + +# @ns_db.route("//k//v/") +# class KeysListVersionsResource(Resource): +# def get(self, nsname, key): +# "Get all versions of a given key in namespace" +# return current_app.backend.key_list_versions(nsname, key) +# + +@app.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_get(nsname: str, key: str): + """Retrieve a given key's value from a given namespace""" + return app.backend.key_get(nsname, key) + + +@app.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_put(nsname: str, key: str, request: Request): + """Create or update the value of a key""" + # check if "reblaze/k/" exists in system/schema-validation + req_json = await request.json() + + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(req_json, schema) + if isValid is False: + raise HTTPException(500, "schema mismatched") + return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + + +@app.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_delete(nsname: str, key: str, request: Request): + """Delete a key""" + return app.backend.key_delete(nsname, key, get_gitactor(request)) + + +# @ns_db.route("//k//") +# class KeyResource(Resource): +# # def get(self, nsname, key): +# # "Retrieve a given key's value from a given namespace" +# # return current_app.backend.key_get(nsname, key) +# +# # def put(self, nsname, key): +# # "Create or update the value of a key" +# # # check if "reblaze/k/" exists in system/schema-validation +# # if nsname != "system": +# # keyName = nsname + "/k/" + key +# # schemas = current_app.backend.key_get("system", "schema-validation") +# # schema = None +# # # find schema if exists and validate the json input +# # for item in schemas.items(): +# # if item[0] == keyName: +# # schema = item[1] +# # break +# # if schema: +# # isValid = validateDbJson(request.json, schema) +# # if isValid is False: +# # abort(500, "schema mismatched") +# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) +# +# # def delete(self, nsname, key): +# # "Delete a key" +# # return current_app.backend.key_delete(nsname, key, get_gitactor()) ############# From 2adf08934bd0b7ec0c46405af0de814b7027f507 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 17 Nov 2022 18:59:07 +0200 Subject: [PATCH 28/55] initial mirroring complete --- .../curieconf/confserver/v3/fast_api.py | 207 ++++++++++++------ 1 file changed, 138 insertions(+), 69 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py index e19c0c6ef..076af7992 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py @@ -13,7 +13,7 @@ jsonschema.Draft4Validator = jsonschema.Draft3Validator # from curieconf import utils -# from curieconf.utils import cloud +from curieconf.utils import cloud # from curieconf.confserver import app @@ -1623,81 +1623,150 @@ async def key_resource_delete(nsname: str, key: str, request: Request): req_fetch_parser.add_argument("url", location="args", help="url to retrieve") -@ns_tools.route("/fetch") -class FetchResource(Resource): - @ns_tools.expect(req_fetch_parser, validate=True) - def get(self): - "Fetch an URL" - args = req_fetch_parser.parse_args() +@app.get("/tools/fetch", tags=[Tags.tools]) +async def fetch_resource_get(url: str): + """Fetch an URL""" + try: + r = requests.get(url) + except Exception as e: + raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) + return r.content + + +# @ns_tools.route("/fetch") +# class FetchResource(Resource): +# @ns_tools.expect(req_fetch_parser, validate=True) +# def get(self): +# "Fetch an URL" +# args = req_fetch_parser.parse_args() +# try: +# r = requests.get(args.url) +# except Exception as e: +# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) +# return make_response(r.content) + + +@app.put("/tools/publish/{config}/", tags=[Tags.tools]) +@app.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) +async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): + """Push configuration to s3 buckets""" + conf = app.backend.configs_get(config, version) + ok = True + status = [] + req_json = await request.json() + if type(req_json) is not list: + raise HTTPException(400, "body must be a list") + for bucket in buckets: + logs = [] try: - r = requests.get(args.url) + cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) except Exception as e: - abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) - return make_response(r.content) - - -@ns_tools.route("/publish//") -@ns_tools.route("/publish//v//") -class PublishResource(Resource): - @ns_tools.expect([m_bucket], validate=True) - def put(self, config, version=None): - "Push configuration to s3 buckets" - conf = current_app.backend.configs_get(config, version) - ok = True - status = [] - if type(request.json) is not list: - abort(400, "body must be a list") - for bucket in request.json: - logs = [] - try: - cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitpush/") -class GitPushResource(Resource): - @ns_tools.expect([m_giturl], validate=True) - def put(self): - "Push git configuration to remote git repositories" - ok = True - status = [] - for giturl in request.json: - try: - current_app.backend.gitpush(giturl["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": giturl["giturl"], "ok": s, "message": msg}) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitfetch/") -class GitFetchResource(Resource): - @ns_tools.expect(m_giturl, validate=True) - def put(self): - "Fetch git configuration from specified remote repository" - ok = True + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + ) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/publish//") +# @ns_tools.route("/publish//v//") +# class PublishResource(Resource): +# @ns_tools.expect([m_bucket], validate=True) +# def put(self, config, version=None): +# "Push configuration to s3 buckets" +# conf = current_app.backend.configs_get(config, version) +# ok = True +# status = [] +# if type(request.json) is not list: +# abort(400, "body must be a list") +# for bucket in request.json: +# logs = [] +# try: +# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) +# except Exception as e: +# ok = False +# s = False +# msg = repr(e) +# else: +# s = True +# msg = "ok" +# status.append( +# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} +# ) +# return make_response({"ok": ok, "status": status}) + + +@app.put("/tools/gitpush/", tags=[Tags.tools]) +async def git_push_resource_put(git_urls: List[GitUrl]): + """Push git configuration to remote git repositories""" + ok = True + status = [] + for giturl in git_urls: try: - current_app.backend.gitfetch(request.json["giturl"]) + app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: - ok = False msg = repr(e) + s = False else: msg = "ok" - return make_response({"ok": ok, "status": msg}) + s = True + status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/gitpush/") +# class GitPushResource(Resource): +# @ns_tools.expect([m_giturl], validate=True) +# def put(self): +# "Push git configuration to remote git repositories" +# ok = True +# status = [] +# for giturl in request.json: +# try: +# current_app.backend.gitpush(giturl["giturl"]) +# except Exception as e: +# msg = repr(e) +# s = False +# else: +# msg = "ok" +# s = True +# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) +# return make_response({"ok": ok, "status": status}) + + +@app.put("/tools/gitfetch/", tags=[Tags.tools]) +async def git_fetch_resource_put(giturl: GitUrl): + """Fetch git configuration from specified remote repository""" + ok = True + try: + app.backend.gitfetch(dict(giturl)["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return {"ok": ok, "status": msg} + + +# @ns_tools.route("/gitfetch/") +# class GitFetchResource(Resource): +# @ns_tools.expect(m_giturl, validate=True) +# def put(self): +# "Fetch git configuration from specified remote repository" +# ok = True +# try: +# current_app.backend.gitfetch(request.json["giturl"]) +# except Exception as e: +# ok = False +# msg = repr(e) +# else: +# msg = "ok" +# return make_response({"ok": ok, "status": msg}) if __name__ == '__main__': From e5bc58aa9fcd8e9ff5f21c5678886a45780fee05 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Sun, 20 Nov 2022 19:33:46 +0200 Subject: [PATCH 29/55] running server, pre checks --- .../server/curieconf/confserver/__init__.py | 38 +- .../server/curieconf/confserver/v3/api.py | 2369 +++++++++++------ curiefense/curieconf/server/setup.py | 1 + 3 files changed, 1579 insertions(+), 829 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 82b540e08..0dbe9486f 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -1,22 +1,15 @@ #! /usr/bin/env python3 import os -import flask -from flask import Flask, current_app from .backend import Backends - -from flask_cors import CORS +import uvicorn +from curieconf.confserver.v3 import api from prometheus_flask_exporter import PrometheusMetrics +from fastapi import FastAPI -## Import all versions -from .v3 import api as api_v3 - -app = Flask(__name__) - -CORS(app, resources={r"/*": {"origins": "*"}}) - -app.register_blueprint(api_v3.api_bp, url_prefix="/api/v3") +app = FastAPI() +app.include_router(api.router) def drop_into_pdb(app, exception): @@ -56,19 +49,16 @@ def main(args=None): options = parser.parse_args(args) - if options.pdb: - flask.got_request_exception.connect(drop_into_pdb) - - metrics = PrometheusMetrics(app) + #TODO - find replacements for got_request_exception and prometheus_flask_exporter + # if options.pdb: + # flask.got_request_exception.connect(drop_into_pdb) + # metrics = PrometheusMetrics(app) try: - with app.app_context(): - current_app.backend = Backends.get_backend(app, options.dbpath) - current_app.options = options.__dict__ - app.run(debug=options.debug, host=options.host, port=options.port) + app.backend = Backends.get_backend(app, options.dbpath) + app.options = options.__dict__ + uvicorn.run(app, host=options.host, port=options.port) + + # app.run(debug=options.debug, host=options.host, port=options.port) finally: pass - - -if __name__ == "__main__": - main() diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index ed1770883..8cc9c0781 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1,25 +1,45 @@ +import datetime +import typing +from enum import Enum +from typing import Optional, List, Union + +from fastapi import FastAPI, Request, HTTPException, APIRouter +from pydantic import BaseModel, Field, validator +import random # needed for generating a random number for an API +import uvicorn # optional if you run it directly from terminal import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type jsonschema.Draft4Validator = jsonschema.Draft3Validator -from flask import Blueprint, request, current_app, abort, make_response -from flask_restx import Resource, Api, fields, marshal, reqparse -from curieconf import utils +# from curieconf import utils from curieconf.utils import cloud +# from curieconf.confserver import app + + +# TODO: TEMP DEFINITIONS +import os + +router = APIRouter() +options = {} +val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) +if val: + options["trusted_username_header"] = val +val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) +if val: + options["trusted_email_header"] = val + import requests from jsonschema import validate from pathlib import Path import json +# api_bp = Blueprint("api_v3", __name__) +# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") -api_bp = Blueprint("api_v3", __name__) -api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - - -ns_configs = api.namespace("configs", description="Configurations") -ns_db = api.namespace("db", description="Database") -ns_tools = api.namespace("tools", description="Tools") +# ns_configs = api.namespace("configs", description="Configurations") +# ns_db = api.namespace("db", description="Database") +# ns_tools = api.namespace("tools", description="Tools") ############## @@ -28,371 +48,628 @@ ### Models for documents +anyTypeUnion = Union[int, float, bool, object, list, None] +anyOp = Optional[object] +anyType = ["number", "string", "boolean", "object", "array", "null"] -class AnyType(fields.Raw): - __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] +# class AnyType(fields.Raw): +# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] # limit -m_threshold = api.model( - "Rate Limit Threshold", - { - "limit": fields.Integer(required=True), - "action": fields.String(required=True), - }, -) - -m_limit = api.model( - "Rate Limit", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "global": fields.Boolean(required=True), - "active": fields.Boolean(required=True), - "timeframe": fields.Integer(required=True), - "thresholds": fields.List(fields.Nested(m_threshold)), - "include": fields.Raw(required=True), - "exclude": fields.Raw(required=True), - "key": AnyType(required=True), - "pairwith": fields.Raw(required=True), - "tags": fields.List(fields.String()), - }, -) +class Threshold(BaseModel): + limit: int + action: str + + +# m_threshold = api.model( +# "Rate Limit Threshold", +# { +# "limit": fields.Integer(required=True), +# "action": fields.String(required=True), +# }, +# ) + +class Limit(BaseModel): + id: str + name: str + description: Optional[str] + _global: bool = Field(alias="global") + active: bool + timeframe: int + thresholds: List[Threshold] + include: typing.Any + exclude: typing.Any + key: anyTypeUnion + pairwith: typing.Any + tags: List[str] + + +# m_limit = api.model( +# "Rate Limit", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "global": fields.Boolean(required=True), +# "active": fields.Boolean(required=True), +# "timeframe": fields.Integer(required=True), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "include": fields.Raw(required=True), +# "exclude": fields.Raw(required=True), +# "key": AnyType(required=True), +# "pairwith": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# }, +# ) # securitypolicy - -m_secprofilemap = api.model( - "Security Profile Map", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "match": fields.String(required=True), - "acl_profile": fields.String(required=True), - "acl_active": fields.Boolean(required=True), - "content_filter_profile": fields.String(required=True), - "content_filter_active": fields.Boolean(required=True), - "limit_ids": fields.List(fields.Raw()), - }, -) - -m_map = api.model( - "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -) - -m_securitypolicy = api.model( - "Security Policy", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "tags": fields.List(fields.String()), - "match": fields.String(required=True), - "session": AnyType(), - "session_ids": AnyType(), - "map": fields.List(fields.Nested(m_secprofilemap)), - }, -) +class SecProfileMap(BaseModel): + id: str + name: str + description: str + match: str + acl_profile: str + acl_active: bool + content_filter_profile: str + content_filter_active: bool + limit_ids: Optional[list] + + +# m_secprofilemap = api.model( +# "Security Profile Map", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.String(required=True), +# "acl_profile": fields.String(required=True), +# "acl_active": fields.Boolean(required=True), +# "content_filter_profile": fields.String(required=True), +# "content_filter_active": fields.Boolean(required=True), +# "limit_ids": fields.List(fields.Raw()), +# }, +# ) + +# TODO = deprecated? +# m_map = api.model( +# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} +# ) + +class SecurityPolicy(BaseModel): + id: str + name: str + description: str + tags: List[str] + match: str + session: anyTypeUnion + session_ids: anyTypeUnion + map: List[SecProfileMap] + + +# m_securitypolicy = api.model( +# "Security Policy", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String()), +# "match": fields.String(required=True), +# "session": AnyType(), +# "session_ids": AnyType(), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# }, +# ) # content filter rule -m_contentfilterrule = api.model( - "Content Filter Rule", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "msg": fields.String(required=True), - "operand": fields.String(required=True), - "severity": fields.Integer(required=True), - "certainity": fields.Integer(required=True), - "category": fields.String(required=True), - "subcategory": fields.String(required=True), - "risk": fields.Integer(required=True), - "tags": fields.List(fields.String()), - "description": fields.String(), - }, -) +class ContentFilterRule(BaseModel): + id: str + name: str + msg: str + operand: str + severity: int + certainity: int + category: str + subcategory: str + risk: int + tags: Optional[List[str]] + description: Optional[str] + + +# m_contentfilterrule = api.model( +# "Content Filter Rule", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "msg": fields.String(required=True), +# "operand": fields.String(required=True), +# "severity": fields.Integer(required=True), +# "certainity": fields.Integer(required=True), +# "category": fields.String(required=True), +# "subcategory": fields.String(required=True), +# "risk": fields.Integer(required=True), +# "tags": fields.List(fields.String()), +# "description": fields.String(), +# }, +# ) # content filter profile - -m_contentfilterprofile = api.model( - "Content Filter Profile", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "ignore_alphanum": fields.Boolean(required=True), - "args": fields.Raw(required=True), - "headers": fields.Raw(required=True), - "cookies": fields.Raw(required=True), - "path": fields.Raw(required=True), - "allsections": fields.Raw(), - "decoding": fields.Raw(required=True), - "masking_seed": fields.String(required=True), - "content_type": fields.List(fields.String()), - "active": fields.List(fields.String()), - "report": fields.List(fields.String()), - "ignore": fields.List(fields.String()), - "tags": fields.List(fields.String()), - "action": fields.String(), - "ignore_body": fields.Boolean(required=True), - }, -) +class ContentFilterProfile(BaseModel): + id: str + name: str + description: Optional[str] + ignore_alphanum: bool + args: typing.Any + headers: typing.Any + cookies: typing.Any + path: typing.Any + allsections: typing.Any + decoding: typing.Any + masking_seed: str + content_type: Optional[List[str]] + active: Optional[List[str]] + report: Optional[List[str]] + ignore: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + ignore_body: bool + + +# m_contentfilterprofile = api.model( +# "Content Filter Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "ignore_alphanum": fields.Boolean(required=True), +# "args": fields.Raw(required=True), +# "headers": fields.Raw(required=True), +# "cookies": fields.Raw(required=True), +# "path": fields.Raw(required=True), +# "allsections": fields.Raw(), +# "decoding": fields.Raw(required=True), +# "masking_seed": fields.String(required=True), +# "content_type": fields.List(fields.String()), +# "active": fields.List(fields.String()), +# "report": fields.List(fields.String()), +# "ignore": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# "ignore_body": fields.Boolean(required=True), +# }, +# ) # aclprofile - -m_aclprofile = api.model( - "ACL Profile", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "allow": fields.List(fields.String()), - "allow_bot": fields.List(fields.String()), - "deny_bot": fields.List(fields.String()), - "passthrough": fields.List(fields.String()), - "deny": fields.List(fields.String()), - "force_deny": fields.List(fields.String()), - "tags": fields.List(fields.String()), - "action": fields.String(), - }, -) +class ACLProfile(BaseModel): + id: str + name: str + description: Optional[str] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + tags: Optional[List[str]] + action: Optional[str] + + +# m_aclprofile = api.model( +# "ACL Profile", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "tags": fields.List(fields.String()), +# "action": fields.String(), +# }, +# ) # Global Filter - -m_globalfilter = api.model( - "Global Filter", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "source": fields.String(required=True), - "mdate": fields.String(required=True), - "description": fields.String(), - "active": fields.Boolean(required=True), - "action": fields.Raw(required=True), - "tags": fields.List(fields.String()), - "rule": AnyType(), - }, -) +class GlobalFilter(BaseModel): + id: str + name: str + source: str + mdate: str + description: str + active: bool + action: typing.Any + tags: Optional[List[str]] + rule: anyTypeUnion + + +# m_glbalfilter = api.model( +# "Global Filter", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "source": fields.String(required=True), +# "mdate": fields.String(required=True), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# "action": fields.Raw(required=True), +# "tags": fields.List(fields.String()), +# "rule": AnyType(), +# }, +# ) # Flow Control -m_flowcontrol = api.model( - "Flow Control", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "timeframe": fields.Integer(required=True), - "key": fields.List(fields.Raw(required=True)), - "sequence": fields.List(fields.Raw(required=True)), - "tags": fields.List(fields.String()), - "include": fields.List(fields.String()), - "exclude": fields.List(fields.String()), - "description": fields.String(), - "active": fields.Boolean(required=True), - }, -) +class FlowControl(BaseModel): + id: str + name: str + timeframe: int + key: List[typing.Any] + sequence: List[typing.Any] + tags: Optional[List[str]] + include: Optional[List[str]] + exclude: Optional[List[str]] + description: Optional[str] + active: bool + + +# +# m_flowcontrol = api.model( +# "Flow Control", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "timeframe": fields.Integer(required=True), +# "key": fields.List(fields.Raw(required=True)), +# "sequence": fields.List(fields.Raw(required=True)), +# "tags": fields.List(fields.String()), +# "include": fields.List(fields.String()), +# "exclude": fields.List(fields.String()), +# "description": fields.String(), +# "active": fields.Boolean(required=True), +# }, +# ) # Action -m_action = api.model( - "Action", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "tags": fields.List(fields.String(required=True)), - "params": fields.Raw(), - "type": fields.String(required=True), - }, -) +class Action(BaseModel): + id: str + name: str + description: Optional[str] + tags: List[str] + params: typing.Any + type: str + + +# m_action = api.model( +# "Action", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "tags": fields.List(fields.String(required=True)), +# "params": fields.Raw(), +# "type": fields.String(required=True), +# }, +# ) # Virtual Tag - -m_virtualtag = api.model( - "Virtual Tag", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - "match": fields.List(fields.Raw(required=True)), - }, -) +class VirtualTag(BaseModel): + id: str + name: str + description: Optional[str] + match: List[typing.Any] + + +# +# m_virtualtag = api.model( +# "Virtual Tag", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# "match": fields.List(fields.Raw(required=True)), +# }, +# ) # custom +class Custom(BaseModel): + id: str + name: str + -m_custom = api.model( - "Custom", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "*": fields.Wildcard(fields.Raw()), - }, -) +# m_custom = api.model( +# "Custom", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) ### mapping from doc name to model models = { - "ratelimits": m_limit, - "securitypolicies": m_securitypolicy, - "contentfilterrules": m_contentfilterrule, - "contentfilterprofiles": m_contentfilterprofile, - "aclprofiles": m_aclprofile, - "globalfilters": m_globalfilter, - "flowcontrol": m_flowcontrol, - "actions": m_action, - "virtualtags": m_custom, - "custom": m_custom, + "ratelimits": Limit, + "securitypolicies": SecurityPolicy, + "contentfilterrules": ContentFilterRule, + "contentfilterprofiles": ContentFilterProfile, + "aclprofiles": ACLProfile, + "globalfilters": GlobalFilter, + "flowcontrol": FlowControl, + "actions": Action, + "virtualtags": Custom, + "custom": Custom, } -### Other models -m_document_mask = api.model( - "Mask for document", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(required=True), - "map": fields.List(fields.Nested(m_secprofilemap)), - "include": fields.Wildcard(fields.Raw()), - "exclude": fields.Wildcard(fields.Raw()), - "tags": fields.List(fields.String()), - "active": fields.Wildcard(fields.Raw()), - "action": fields.Raw(), - "sequence": fields.List(fields.Raw()), - "timeframe": fields.Integer(), - "thresholds": fields.List(fields.Nested(m_threshold)), - "pairwith": fields.Raw(), - "content_type": fields.List(fields.String()), - "params": fields.Raw(), - "decoding": fields.Raw(), - "category": fields.String(), - "subcategory": fields.String(), - "risk": fields.Integer(), - "allow": fields.List(fields.String()), - "allow_bot": fields.List(fields.String()), - "deny_bot": fields.List(fields.String()), - "passthrough": fields.List(fields.String()), - "deny": fields.List(fields.String()), - "force_deny": fields.List(fields.String()), - "match": fields.String(), - "type": fields.String(), - "*": fields.Wildcard(fields.Raw()), - }, -) - -m_version_log = api.model( - "Version log", - { - "version": fields.String(), - "date": fields.DateTime(dt_format="iso8601"), - "*": fields.Wildcard(fields.Raw()), - }, -) - -m_meta = api.model( - "Meta", - { - "id": fields.String(required=True), - "description": fields.String(required=True), - "date": fields.DateTime(), - "logs": fields.List(fields.Nested(m_version_log), default=[]), - "version": fields.String(), - }, -) - -m_blob_entry = api.model( - "Blob Entry", - { - "format": fields.String(required=True), - "blob": AnyType(), - }, -) - -m_blob_list_entry = api.model( - "Blob ListEntry", - { - "name": fields.String(), - }, -) - -m_document_list_entry = api.model( - "Document ListEntry", - { - "name": fields.String(), - "entries": fields.Integer(), - }, -) - - -m_config_documents = api.model( - "Config Documents", - {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -) - -m_config_blobs = api.model( - "Config Blobs", - {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -) - -m_config_delete_blobs = api.model( - "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -) - -m_config = api.model( - "Config", - { - "meta": fields.Nested(m_meta, default={}), - "documents": fields.Nested(m_config_documents, default={}), - "blobs": fields.Nested(m_config_blobs, default={}), - "delete_documents": fields.Nested(m_config_documents, default={}), - "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), - }, -) - -m_edit = api.model( - "Edit", - { - "path": fields.String(required=True), - "value": fields.String(required=True), - }, -) - -m_basic_entry = api.model( - "Basic Document Entry", - { - "id": fields.String(required=True), - "name": fields.String(required=True), - "description": fields.String(), - }, -) +### Other models +class DocumentMask(BaseModel): + id: str + name: str + description: str + map: Optional[List[SecProfileMap]] + include: Optional[List[typing.Any]] + exclude: Optional[List[typing.Any]] + tags: Optional[List[str]] + active: Optional[List[typing.Any]] + action: typing.Any + sequence: Optional[List[typing.Any]] + timeframe: Optional[int] + thresholds: Optional[List[Threshold]] + pairwith: typing.Any + content_type: Optional[List[str]] + params: typing.Any + decoding: typing.Any + category: Optional[str] + subcategory: Optional[str] + risk: Optional[int] + allow: Optional[List[str]] + allow_bot: Optional[List[str]] + deny_bot: Optional[List[str]] + passthrough: Optional[List[str]] + deny: Optional[List[str]] + force_deny: Optional[List[str]] + match: Optional[str] = "j" + _type: Optional[str] = Field(alias="type") + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# m_document_mask = api.model( +# "Mask for document", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(required=True), +# "map": fields.List(fields.Nested(m_secprofilemap)), +# "include": fields.Wildcard(fields.Raw()), +# "exclude": fields.Wildcard(fields.Raw()), +# "tags": fields.List(fields.String()), +# "active": fields.Wildcard(fields.Raw()), +# "action": fields.Raw(), +# "sequence": fields.List(fields.Raw()), +# "timeframe": fields.Integer(), +# "thresholds": fields.List(fields.Nested(m_threshold)), +# "pairwith": fields.Raw(), +# "content_type": fields.List(fields.String()), +# "params": fields.Raw(), +# "decoding": fields.Raw(), +# "category": fields.String(), +# "subcategory": fields.String(), +# "risk": fields.Integer(), +# "allow": fields.List(fields.String()), +# "allow_bot": fields.List(fields.String()), +# "deny_bot": fields.List(fields.String()), +# "passthrough": fields.List(fields.String()), +# "deny": fields.List(fields.String()), +# "force_deny": fields.List(fields.String()), +# "match": fields.String(), +# "type": fields.String(), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class VersionLog(BaseModel): + version: Optional[str] + # TODO - dt_format="iso8601" + date: Optional[datetime.datetime] + _star: Optional[List[typing.Any]] = Field(alias="*") + + +# +# m_version_log = api.model( +# "Version log", +# { +# "version": fields.String(), +# "date": fields.DateTime(dt_format="iso8601"), +# "*": fields.Wildcard(fields.Raw()), +# }, +# ) + +class Meta(BaseModel): + id: str + description: str + date: Optional[datetime.datetime] + logs: Optional[List[VersionLog]] = [] + version: Optional[str] + + +# m_meta = api.model( +# "Meta", +# { +# "id": fields.String(required=True), +# "description": fields.String(required=True), +# "date": fields.DateTime(), +# "logs": fields.List(fields.Nested(m_version_log), default=[]), +# "version": fields.String(), +# }, +# ) + +class BlobEntry(BaseModel): + format: str + blob: anyTypeUnion + + +# m_blob_entry = api.model( +# "Blob Entry", +# { +# "format": fields.String(required=True), +# "blob": AnyType(), +# }, +# ) + +class BlobListEntry(BaseModel): + name: Optional[str] + + +# m_blob_list_entry = api.model( +# "Blob ListEntry", +# { +# "name": fields.String(), +# }, +# ) + +class DocumentListEntry(BaseModel): + name: Optional[str] + entries: Optional[int] + + +# m_document_list_entry = api.model( +# "Document ListEntry", +# { +# "name": fields.String(), +# "entries": fields.Integer(), +# }, +# ) + +class ConfigDocuments(BaseModel): + ratelimits: Optional[List[models["ratelimits"]]] = [] + securitypolicies: Optional[List[models["securitypolicies"]]] = [] + contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] + contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] + aclprofiles: Optional[List[models["aclprofiles"]]] = [] + globalfilters: Optional[List[models["globalfilters"]]] = [] + flowcontrol: Optional[List[models["flowcontrol"]]] = [] + actions: Optional[List[models["actions"]]] = [] + virtualtags: Optional[List[models["virtualtags"]]] = [] + custom: Optional[List[models["custom"]]] = [] + + +# m_config_documents = api.model( +# "Config Documents", +# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, +# ) + + +class ConfigBlobs(BaseModel): + geolite2asn: Optional[List[Optional[BlobEntry]]] + geolite2country: Optional[List[Optional[BlobEntry]]] + geolite2city: Optional[List[Optional[BlobEntry]]] + customconf: Optional[List[Optional[BlobEntry]]] + + +# m_config_blobs = api.model( +# "Config Blobs", +# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, +# ) + +class ConfigDeleteBlobs(BaseModel): + geolite2asn: Optional[bool] + geolite2country: Optional[bool] + geolite2city: Optional[bool] + customconf: Optional[bool] + + +# m_config_delete_blobs = api.model( +# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} +# ) + +class Config(BaseModel): + meta: Meta = {} + documents: ConfigDocuments = {} + blobs: ConfigBlobs = {} + delete_documents: ConfigDocuments = {} + delete_blobs: ConfigDeleteBlobs = {} + + +# m_config = api.model( +# "Config", +# { +# "meta": fields.Nested(m_meta, default={}), +# "documents": fields.Nested(m_config_documents, default={}), +# "blobs": fields.Nested(m_config_blobs, default={}), +# "delete_documents": fields.Nested(m_config_documents, default={}), +# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), +# }, +# ) + +class Edit(BaseModel): + path: str + value: str + + +# m_edit = api.model( +# "Edit", +# { +# "path": fields.String(required=True), +# "value": fields.String(required=True), +# }, +# ) + +class BasicEntry(BaseModel): + id: str + name: str + description: Optional[str] + + +# m_basic_entry = api.model( +# "Basic Document Entry", +# { +# "id": fields.String(required=True), +# "name": fields.String(required=True), +# "description": fields.String(), +# }, +# ) ### Publish -m_bucket = api.model( - "Bucket", - { - "name": fields.String(required=True), - "url": fields.String(required=True), - }, -) +class Bucket(BaseModel): + name: str + url: str + + +# m_bucket = api.model( +# "Bucket", +# { +# "name": fields.String(required=True), +# "url": fields.String(required=True), +# }, +# ) ### Git push & pull -m_giturl = api.model( - "GitUrl", - { - "giturl": fields.String(required=True), - }, -) +class GitUrl(BaseModel): + giturl: str + +# m_giturl = api.model( +# "GitUrl", +# { +# "giturl": fields.String(required=True), +# }, +# ) ### Db +class DB(BaseModel): + pass + + +# m_db = api.model("db", {}) -m_db = api.model("db", {}) ### Document Schema validation @@ -421,15 +698,15 @@ def validateDbJson(json_data, schema): ### Set git actor according to config & defined HTTP headers -def get_gitactor(): +def get_gitactor(request): email, username = "", "" - email_header = current_app.options.get("trusted_email_header", None) + email_header = app.options.get("trusted_email_header", None) if email_header: email = request.headers.get(email_header, "") - username_header = current_app.options.get("trusted_username_header", None) + username_header = app.options.get("trusted_username_header", None) if username_header: username = request.headers.get(username_header, "") - return current_app.backend.prepare_actor(username, email) + return app.backend.prepare_actor(username, email) base_path = Path(__file__).parent @@ -444,7 +721,7 @@ def get_gitactor(): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -455,7 +732,7 @@ def get_gitactor(): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -468,8 +745,6 @@ def get_gitactor(): custom_file_path = (base_path / "./json/custom.schema").resolve() with open(custom_file_path) as json_file: custom_schema = json.load(json_file) - - schema_type_map = { "ratelimits": ratelimits_schema, "securitypolicies": securitypolicies_schema, @@ -484,87 +759,163 @@ def get_gitactor(): } +class Tags(Enum): + congifs = "configs" + db = "db" + tools = "tools" + + ################ ### CONFIGS ### ################ - -@ns_configs.route("/") -class Configs(Resource): - @ns_configs.marshal_list_with(m_meta, skip_none=True) - def get(self): - "Get the detailed list of existing configurations" - return current_app.backend.configs_list() - - @ns_configs.expect(m_config, validate=True) - def post(self): - "Create a new configuration" - data = request.json - return current_app.backend.configs_create(data, get_gitactor()) - - -@ns_configs.route("//") -class Config(Resource): - @ns_configs.marshal_with(m_config, skip_none=True) - def get(self, config): - "Retrieve a complete configuration" - return current_app.backend.configs_get(config) - - @ns_configs.expect(m_config, validate=True) - def post(self, config): - "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = request.json - return current_app.backend.configs_create(data, config, get_gitactor()) - - @ns_configs.expect(m_meta, validate=True) - def put(self, config): - "Update an existing configuration" - data = request.json - return current_app.backend.configs_update(config, data, get_gitactor()) - - def delete(self, config): - "Delete a configuration" - return current_app.backend.configs_delete(config) - - -@ns_configs.route("//clone/") -class ConfigClone(Resource): - @ns_configs.expect(m_meta, validate=True) - def post(self, config): - "Clone a configuration. New name is provided in POST data" - data = request.json - return current_app.backend.configs_clone(config, data) - - -@ns_configs.route("//clone//") -class ConfigCloneName(Resource): - @ns_configs.expect(m_meta, validate=True) - def post(self, config, new_name): - "Clone a configuration. New name is provided URL" - data = request.json - return current_app.backend.configs_clone(config, data, new_name) - - -@ns_configs.route("//v/") -class ConfigListVersion(Resource): - @ns_configs.marshal_with(m_version_log, skip_none=True) - def get(self, config): - "Get all versions of a given configuration" - return current_app.backend.configs_list_versions(config) - - -@ns_configs.route("//v//") -class ConfigVersion(Resource): - def get(self, config, version): - "Retrieve a specific version of a configuration" - return current_app.backend.configs_get(config, version) - - -@ns_configs.route("//v//revert/") -class ConfigRevert(Resource): - def put(self, config, version): - "Create a new version for a configuration from an old version" - return current_app.backend.configs_revert(config, version, get_gitactor()) +@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta]) +async def configs_get(request: Request): + """Get the detailed list of existing configurations""" + res = request.app.backend.configs_list() + return res + + +@router.post("/configs/", tags=[Tags.congifs]) +async def configs_post(config: Config, request: Request): + """Create a new configuration""" + data = dict(config) + return request.app.backend.configs_create(data, get_gitactor(request)) + + +# +# @ns_configs.route("/") +# class Configs(Resource): +# # @ns_configs.marshal_list_with(m_meta, skip_none=True) +# # def get(self): +# # "Get the detailed list of existing configurations" +# # return current_app.backend.configs_list() +# +# @ns_configs.expect(m_config, validate=True) +# def post(self): +# "Create a new configuration" +# data = request.json +# return current_app.backend.configs_create(data, get_gitactor()) + +@router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +async def config_get(config: str): + """Retrieve a complete configuration""" + return app.backend.configs_get(config) + + +@router.post("/configs/{config}/", tags=[Tags.congifs]) +async def config_post(config: str, m_config: Config, request: Request): + "Create a new configuration. Configuration name in URL overrides configuration in POST data" + data = dict(m_config) + return app.backend.configs_create(data, config, get_gitactor(request)) + + +@router.put("/configs/{config}/", tags=[Tags.congifs]) +async def config_put(config: str, meta: Meta, request: Request): + """Update an existing configuration""" + data = dict(meta) + return app.backend.configs_update(config, data, get_gitactor(request)) + + +@router.delete("/configs/{config}/", tags=[Tags.congifs]) +async def config_delete(config: str): + """Delete a configuration""" + return app.backend.configs_delete(config) + + +# @ns_configs.route("//") +# class Config(Resource): +# # @ns_configs.marshal_with(m_config, skip_none=True) +# # def get(self, config): +# # "Retrieve a complete configuration" +# # return current_app.backend.configs_get(config) +# +# # @ns_configs.expect(m_config, validate=True) +# # def post(self, config): +# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" +# # data = request.json +# # return current_app.backend.configs_create(data, config, get_gitactor()) +# +# # @ns_configs.expect(m_meta, validate=True) +# # def put(self, config): +# # "Update an existing configuration" +# # data = request.json +# # return current_app.backend.configs_update(config, data, get_gitactor()) +# +# def delete(self, config): +# "Delete a configuration" +# return current_app.backend.configs_delete(config) + + +@router.post("/configs/{config}/clone/", tags=[Tags.congifs]) +async def config_clone_post(config: str, meta: Meta): + """Clone a configuration. New name is provided in POST data""" + data = dict(meta) + return app.backend.configs_clone(config, data) + + +# @ns_configs.route("//clone/") +# class ConfigClone(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config): +# "Clone a configuration. New name is provided in POST data" +# data = request.json +# return current_app.backend.configs_clone(config, data) +# + +@router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) +async def config_clone_name_post(config: str, new_name: str, meta: Meta): + """Clone a configuration. New name is provided URL""" + data = dict(meta) + return app.backend.configs_clone(config, data, new_name) + + +# @ns_configs.route("//clone//") +# class ConfigCloneName(Resource): +# @ns_configs.expect(m_meta, validate=True) +# def post(self, config, new_name): +# "Clone a configuration. New name is provided URL" +# data = request.json +# return current_app.backend.configs_clone(config, data, new_name) + + +@router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def config_list_version_get(config: str): + """Get all versions of a given configuration""" + return app.backend.configs_list_versions(config) + + +# @ns_configs.route("//v/") +# class ConfigListVersion(Resource): +# @ns_configs.marshal_with(m_version_log, skip_none=True) +# def get(self, config): +# "Get all versions of a given configuration" +# return current_app.backend.configs_list_versions(config) + + +@router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +async def config_version_get(config: str, version: str): + """Retrieve a specific version of a configuration""" + return app.backend.configs_get(config, version) + + +# @ns_configs.route("//v//") +# class ConfigVersion(Resource): +# def get(self, config, version): +# "Retrieve a specific version of a configuration" +# return current_app.backend.configs_get(config, version) + +@router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +async def config_revert_put(config: str, version: str, request: Request): + """Create a new version for a configuration from an old version""" + return app.backend.configs_revert(config, version, get_gitactor(request)) + + +# @ns_configs.route("//v//revert/") +# class ConfigRevert(Resource): +# def put(self, config, version): +# "Create a new version for a configuration from an old version" +# return current_app.backend.configs_revert(config, version, get_gitactor()) ############# @@ -572,356 +923,695 @@ def put(self, config, version): ############# -@ns_configs.route("//b/") -class BlobsResource(Resource): - @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) - def get(self, config): - "Retrieve the list of available blobs" - res = current_app.backend.blobs_list(config) - return res - - -@ns_configs.route("//b//") -class BlobResource(Resource): - @ns_configs.marshal_with(m_blob_entry, skip_none=True) - def get(self, config, blob): - "Retrieve a blob" - return current_app.backend.blobs_get(config, blob) - - @ns_configs.expect(m_blob_entry, validate=True) - def post(self, config, blob): - "Create a new blob" - return current_app.backend.blobs_create( - config, blob, request.json, get_gitactor() - ) - - @ns_configs.expect(m_blob_entry, validate=True) - def put(self, config, blob): - "Replace a blob with new data" - return current_app.backend.blobs_update( - config, blob, request.json, get_gitactor() - ) - - def delete(self, config, blob): - "Delete a blob" - return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@ns_configs.route("//b//v/") -class BlobListVersionResource(Resource): - @ns_configs.marshal_list_with(m_version_log, skip_none=True) - def get(self, config, blob): - "Retrieve the list of versions of a given blob" - res = current_app.backend.blobs_list_versions(config, blob) - return res - - -@ns_configs.route("//b//v//") -class BlobVersionResource(Resource): - @ns_configs.marshal_list_with(m_version_log, skip_none=True) - def get(self, config, blob, version): - "Retrieve the given version of a blob" - return current_app.backend.blobs_get(config, blob, version) - - -@ns_configs.route("//b//v//revert/") -class BlobRevertResource(Resource): - def put(self, config, blob, version): - "Create a new version for a blob from an old version" - return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) +@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +async def blobs_resource_get(config: str): + """Retrieve the list of available blobs""" + res = app.backend.blobs_list(config) + return res + + +# @ns_configs.route("//b/") +# class BlobsResource(Resource): +# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of available blobs" +# res = current_app.backend.blobs_list(config) +# return res + +@router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +async def blob_resource_get(config: str, blob: str): + """Retrieve a blob""" + return app.backend.blobs_get(config, blob) + + +@router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_create( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): + """Create a new blob""" + return app.backend.blobs_update( + config, blob, dict(blob_entry), get_gitactor(request) + ) + + +@router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_get(config: str, blob: str, request: Request): + """Delete a blob""" + return app.backend.blobs_delete(config, blob, get_gitactor(request)) + + +# +# @ns_configs.route("//b//") +# class BlobResource(Resource): +# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) +# # def get(self, config, blob): +# # "Retrieve a blob" +# # return current_app.backend.blobs_get(config, blob) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def post(self, config, blob): +# # "Create a new blob" +# # return current_app.backend.blobs_create( +# # config, blob, request.json, get_gitactor() +# # ) +# +# # @ns_configs.expect(m_blob_entry, validate=True) +# # def put(self, config, blob): +# # "Replace a blob with new data" +# # return current_app.backend.blobs_update( +# # config, blob, request.json, get_gitactor() +# # ) +# +# def delete(self, config, blob): +# "Delete a blob" +# return current_app.backend.blobs_delete(config, blob, get_gitactor()) + + +@router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +async def blob_list_version_resource_get(config: str, blob: str): + "Retrieve the list of versions of a given blob" + res = app.backend.blobs_list_versions(config, blob) + return res + + +# @ns_configs.route("//b//v/") +# class BlobListVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob): +# "Retrieve the list of versions of a given blob" +# res = current_app.backend.blobs_list_versions(config, blob) +# return res + + +@router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +async def blob_version_resource_get(config: str, blob: str, version: str): + """Retrieve the given version of a blob""" + return app.backend.blobs_get(config, blob, version) + + +# @ns_configs.route("//b//v//") +# class BlobVersionResource(Resource): +# @ns_configs.marshal_list_with(m_version_log, skip_none=True) +# def get(self, config, blob, version): +# "Retrieve the given version of a blob" +# return current_app.backend.blobs_get(config, blob, version) + +@router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): + """Create a new version for a blob from an old version""" + return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + + +# +# @ns_configs.route("//b//v//revert/") +# class BlobRevertResource(Resource): +# def put(self, config, blob, version): +# "Create a new version for a blob from an old version" +# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) ################# ### DOCUMENTS ### ################# - -@ns_configs.route("//d/") -class DocumentsResource(Resource): - @ns_configs.marshal_with(m_document_list_entry, skip_none=True) - def get(self, config): - "Retrieve the list of existing documents in this configuration" - res = current_app.backend.documents_list(config) - return res - - -@ns_configs.route("//d//") -class DocumentResource(Resource): - @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) - def get(self, config, document): - "Get a complete document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect([m_basic_entry], validate=True) - def post(self, config, document): - "Create a new complete document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched: \n" + err) - res = current_app.backend.documents_create( - config, document, data, get_gitactor() - ) - return res - - @ns_configs.expect([m_basic_entry], validate=True) - def put(self, config, document): - "Update an existing document" - if document not in models: - abort(404, "document does not exist") - data = marshal(request.json, models[document], skip_none=True) - for entry in request.json: - isValid, err = validateJson(entry, document) - if isValid is False: - abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = current_app.backend.documents_update( - config, document, data, get_gitactor() - ) - return res - - def delete(self, config, document): - "Delete/empty a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_delete(config, document, get_gitactor()) - return res - - -@ns_configs.route("//d//v/") -class DocumentListVersionResource(Resource): - def get(self, config, document): - "Retrieve the existing versions of a given document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_list_versions(config, document) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route("//d//v//") -class DocumentVersionResource(Resource): - def get(self, config, document, version): - "Get a given version of a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.documents_get(config, document, version) - return marshal(res, models[document], skip_none=True) - - -@ns_configs.route("//d//v//revert/") -class DocumentRevertResource(Resource): - def put(self, config, document, version): - "Create a new version for a document from an old version" - return current_app.backend.documents_revert( - config, document, version, get_gitactor() - ) +@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +async def document_resource(config: str): + """Retrieve the list of existing documents in this configuration""" + res = app.backend.documents_list(config) + return res + + +# +# @ns_configs.route("//d/") +# class DocumentsResource(Resource): +# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) +# def get(self, config): +# "Retrieve the list of existing documents in this configuration" +# res = current_app.backend.documents_list(config) +# return res + + +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +async def document_resource_get(config: str, document: str): + """Get a complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + res = app.backend.documents_get(config, document) + res = {key: res[key] for key in list(models[document].__fields__.keys())} + return res + + +async def _filter(data, keys): + return {key: data[key] for key in keys} + + +@router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Create a new complete document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched: \n" + err) + res = app.backend.documents_create( + config, document, data, get_gitactor(request) + ) + return res + + +@router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): + """Update an existing document""" + if document not in models: + raise HTTPException(status_code=404, detail="document does not exist") + + data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + for entry in basic_entries: + isValid, err = validateJson(dict(entry), document) + if isValid is False: + raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + res = app.backend.documents_update( + config, document, data, get_gitactor(request) + ) + return res + + +@router.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) +async def document_resource_delete(config: str, document: str, request: Request): + """Delete/empty a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_delete(config, document, get_gitactor(request)) + return res + + +# @ns_configs.route("//d//") +# class DocumentResource(Resource): +# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) +# # def get(self, config, document): +# # "Get a complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_get(config, document) +# # return marshal(res, models[document], skip_none=True) +# # +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def post(self, config, document): +# # "Create a new complete document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched: \n" + err) +# # res = current_app.backend.documents_create( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # @ns_configs.expect([m_basic_entry], validate=True) +# # def put(self, config, document): +# # "Update an existing document" +# # if document not in models: +# # abort(404, "document does not exist") +# # data = marshal(request.json, models[document], skip_none=True) +# # for entry in request.json: +# # isValid, err = validateJson(entry, document) +# # if isValid is False: +# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) +# # res = current_app.backend.documents_update( +# # config, document, data, get_gitactor() +# # ) +# # return res +# +# # def delete(self, config, document): +# # "Delete/empty a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.documents_delete(config, document, get_gitactor()) +# # return res + + +@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) +async def document_list_version_resource_get(config: str, document: str): + """Retrieve the existing versions of a given document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_list_versions(config, document) + res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res + + +# +# @ns_configs.route("//d//v/") +# class DocumentListVersionResource(Resource): +# def get(self, config, document): +# "Retrieve the existing versions of a given document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_list_versions(config, document) +# return marshal(res, m_version_log, skip_none=True) + + +@router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) +async def document_version_resource_get(config: str, document: str, version: str): + """Get a given version of a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.documents_get(config, document, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route("//d//v//") +# class DocumentVersionResource(Resource): +# def get(self, config, document, version): +# "Get a given version of a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.documents_get(config, document, version) +# return marshal(res, models[document], skip_none=True) + +@router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) +async def document_revert_resource_put(config: str, document: str, version: str, request: Request): + """Create a new version for a document from an old version""" + return app.backend.documents_revert( + config, document, version, get_gitactor(request) + ) + + +# @ns_configs.route("//d//v//revert/") +# class DocumentRevertResource(Resource): +# def put(self, config, document, version): +# "Create a new version for a document from an old version" +# return current_app.backend.documents_revert( +# config, document, version, get_gitactor() +# ) ############### ### ENTRIES ### ############### - -@ns_configs.route("//d//e/") -class EntriesResource(Resource): - def get(self, config, document): - "Retrieve the list of entries in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list(config, document) - return res # XXX: marshal - - @ns_configs.expect(m_basic_entry, validate=True) - def post(self, config, document): - "Create an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_create( - config, document, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - -@ns_configs.route("//d//e//") -class EntryResource(Resource): - def get(self, config, document, entry): - "Retrieve an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry) - return marshal(res, models[document], skip_none=True) - - @ns_configs.expect(m_basic_entry, validate=True) - def put(self, config, document, entry): - "Update an entry in a document" - if document not in models: - abort(404, "document does not exist") - isValid, err = validateJson(request.json, document) - if isValid: - data = marshal(request.json, models[document], skip_none=True) - res = current_app.backend.entries_update( - config, document, entry, data, get_gitactor() - ) - return res - else: - abort(500, "schema mismatched: \n" + err) - - def delete(self, config, document, entry): - "Delete an entry from a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_delete( - config, document, entry, get_gitactor() +@router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_get(config: str, document: str): + """Retrieve the list of entries in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list(config, document) + return res # XXX: marshal + + +@router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) +async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): + "Create an entry in a document" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + res = app.backend.entries_create( + config, document, data, get_gitactor(request) ) return res - - -@ns_configs.route("//d//e//v/") -class EntryListVersionResource(Resource): - def get(self, config, document, entry): - "Get the list of existing versions of a given entry in a document" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_list_versions(config, document, entry) - return marshal(res, m_version_log, skip_none=True) - - -@ns_configs.route( - "//d//e//v//" -) -class EntryVersionResource(Resource): - def get(self, config, document, entry, version): - "Get a given version of a document entry" - if document not in models: - abort(404, "document does not exist") - res = current_app.backend.entries_get(config, document, entry, version) - return marshal(res, models[document], skip_none=True) + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +# @ns_configs.route("//d//e/") +# class EntriesResource(Resource): +# # def get(self, config, document): +# # "Retrieve the list of entries in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_list(config, document) +# # return res # XXX: marshal +# +# @ns_configs.expect(m_basic_entry, validate=True) +# def post(self, config, document): +# "Create an entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# isValid, err = validateJson(request.json, document) +# if isValid: +# data = marshal(request.json, models[document], skip_none=True) +# res = current_app.backend.entries_create( +# config, document, data, get_gitactor() +# ) +# return res +# else: +# abort(500, "schema mismatched: \n" + err) + + +@router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_get(config: str, document: str, entry: str): + """Retrieve an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry) + return {key: res for key in list(models[document].__fields__.keys())} + + +@router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): + """Update an entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + isValid, err = validateJson(dict(basic_entry), document) + if isValid: + data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + + res = app.backend.entries_update( + config, document, entry, data, get_gitactor(request) + ) + return res + else: + raise HTTPException(500, "schema mismatched: \n" + err) + + +@router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) +async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): + """Delete an entry from a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_delete( + config, document, entry, get_gitactor(request) + ) + return res + + +# @ns_configs.route("//d//e//") +# class EntryResource(Resource): +# # def get(self, config, document, entry): +# # "Retrieve an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_get(config, document, entry) +# # return marshal(res, models[document], skip_none=True) +# +# # @ns_configs.expect(m_basic_entry, validate=True) +# # def put(self, config, document, entry): +# # "Update an entry in a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # isValid, err = validateJson(request.json, document) +# # if isValid: +# # data = marshal(request.json, models[document], skip_none=True) +# # res = current_app.backend.entries_update( +# # config, document, entry, data, get_gitactor() +# # ) +# # return res +# # else: +# # abort(500, "schema mismatched: \n" + err) +# +# # def delete(self, config, document, entry): +# # "Delete an entry from a document" +# # if document not in models: +# # abort(404, "document does not exist") +# # res = current_app.backend.entries_delete( +# # config, document, entry, get_gitactor() +# # ) +# # return res + + +@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) +async def entry_list_version_resource_get(config: str, document: str, entry: str): + """Get the list of existing versions of a given entry in a document""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_list_versions(config, document, entry) + return {key: res[key] for key in list(VersionLog.__fields__.keys())} + + +# +# @ns_configs.route("//d//e//v/") +# class EntryListVersionResource(Resource): +# def get(self, config, document, entry): +# "Get the list of existing versions of a given entry in a document" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_list_versions(config, document, entry) +# return marshal(res, m_version_log, skip_none=True) + + +@router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) +async def entry_version_resource_get(config: str, document: str, entry: str, version: str): + """Get a given version of a document entry""" + if document not in models: + raise HTTPException(404, "document does not exist") + res = app.backend.entries_get(config, document, entry, version) + return {key: res[key] for key in list(models[document].__fields__.keys())} + + +# @ns_configs.route( +# "//d//e//v//" +# ) +# class EntryVersionResource(Resource): +# def get(self, config, document, entry, version): +# "Get a given version of a document entry" +# if document not in models: +# abort(404, "document does not exist") +# res = current_app.backend.entries_get(config, document, entry, version) +# return marshal(res, models[document], skip_none=True) ################ ### Database ### ################ +@router.get("/db/", tags=[Tags.db]) +async def db_resource_get(): + """Get the list of existing namespaces""" + return app.backend.ns_list() -@ns_db.route("/") -class DbResource(Resource): - def get(self): - "Get the list of existing namespaces" - return current_app.backend.ns_list() +# @ns_db.route("/") +# class DbResource(Resource): +# def get(self): +# "Get the list of existing namespaces" +# return current_app.backend.ns_list() -@ns_db.route("/v/") -class DbQueryResource(Resource): - def get(self): - "List all existing versions of namespaces" - return current_app.backend.ns_list_versions() +@router.get("/db/v/", tags=[Tags.db]) +async def db_query_resource_get(): + """List all existing versions of namespaces""" + return app.backend.ns_list_versions() -@ns_db.route("//") -class NSResource(Resource): - def get(self, nsname): - "Get a complete namespace" - try: - return current_app.backend.ns_get(nsname, version=None) - except KeyError: - abort(404, "namespace [%s] does not exist" % nsname) - @ns_db.expect(m_db, validate=True) - def post(self, nsname): - "Create a non-existing namespace from data" - try: - return current_app.backend.ns_create(nsname, request.json, get_gitactor()) - except Exception: - abort(409, "namespace [%s] already exists" % nsname) +# +# @ns_db.route("/v/") +# class DbQueryResource(Resource): +# def get(self): +# "List all existing versions of namespaces" +# return current_app.backend.ns_list_versions() - @ns_db.expect(m_db, validate=True) - def put(self, nsname): - "Merge data into a namespace" - return current_app.backend.ns_update(nsname, request.json, get_gitactor()) - def delete(self, nsname): - "Delete an existing namespace" - try: - return current_app.backend.ns_delete(nsname, get_gitactor()) - except KeyError: - abort(409, "namespace [%s] does not exist" % nsname) +@router.get("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_get(nsname: str): + """Get a complete namespace""" + try: + return app.backend.ns_get(nsname, version=None) + except KeyError: + raise HTTPException(404, "namespace [%s] does not exist" % nsname) -@ns_db.route("//v//") -class NSVersionResource(Resource): - def get(self, nsname, version): - "Get a given version of a namespace" - return current_app.backend.ns_get(nsname, version) +@router.post("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_post(nsname: str, db: DB, request: Request): + """Create a non-existing namespace from data""" + try: + return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + except Exception: + raise HTTPException(409, "namespace [%s] already exists" % nsname) -@ns_db.route("//v//revert/") -class NSVersionResource(Resource): - def put(self, nsname, version): - "Create a new version for a namespace from an old version" - try: - return current_app.backend.ns_revert(nsname, version, get_gitactor()) - except KeyError: - abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@ns_db.route("//q/") -class NSQueryResource(Resource): - def post(self, nsname): - "Run a JSON query on the namespace and returns the results" - return current_app.backend.ns_query(nsname, request.json) - - -@ns_db.route("//k/") -class KeysResource(Resource): - def get(self, nsname): - "List all keys of a given namespace" - return current_app.backend.key_list(nsname) - - -@ns_db.route("//k//v/") -class KeysListVersionsResource(Resource): - def get(self, nsname, key): - "Get all versions of a given key in namespace" - return current_app.backend.key_list_versions(nsname, key) - - -@ns_db.route("//k//") -class KeyResource(Resource): - def get(self, nsname, key): - "Retrieve a given key's value from a given namespace" - return current_app.backend.key_get(nsname, key) - - def put(self, nsname, key): - "Create or update the value of a key" - # check if "reblaze/k/" exists in system/schema-validation - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = current_app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(request.json, schema) - if isValid is False: - abort(500, "schema mismatched") - return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) - - def delete(self, nsname, key): - "Delete a key" - return current_app.backend.key_delete(nsname, key, get_gitactor()) +@router.put("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, db: DB, request: Request): + """Merge data into a namespace""" + return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + + +@router.delete("/db/{nsname}/", tags=[Tags.db]) +async def ns_resource_put(nsname: str, request: Request): + """Delete an existing namespace""" + try: + return app.backend.ns_delete(nsname, get_gitactor(request)) + except KeyError: + raise HTTPException(409, "namespace [%s] does not exist" % nsname) + + +# @ns_db.route("//") +# class NSResource(Resource): +# # def get(self, nsname): +# # "Get a complete namespace" +# # try: +# # return current_app.backend.ns_get(nsname, version=None) +# # except KeyError: +# # abort(404, "namespace [%s] does not exist" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def post(self, nsname): +# # "Create a non-existing namespace from data" +# # try: +# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) +# # except Exception: +# # abort(409, "namespace [%s] already exists" % nsname) +# +# # @ns_db.expect(m_db, validate=True) +# # def put(self, nsname): +# # "Merge data into a namespace" +# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) +# +# # def delete(self, nsname): +# # "Delete an existing namespace" +# # try: +# # return current_app.backend.ns_delete(nsname, get_gitactor()) +# # except KeyError: +# # abort(409, "namespace [%s] does not exist" % nsname) + + +@router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +async def ns_version_resource_get(nsname: str, version: str): + """Get a given version of a namespace""" + return app.backend.ns_get(nsname, version) + + +# @ns_db.route("//v//") +# class NSVersionResource(Resource): +# def get(self, nsname, version): +# "Get a given version of a namespace" +# return current_app.backend.ns_get(nsname, version) + + +@router.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) +async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): + """Create a new version for a namespace from an old version""" + try: + return app.backend.ns_revert(nsname, version, get_gitactor(request)) + except KeyError: + raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +# +# @ns_db.route("//v//revert/") +# class NSVersionResource(Resource): +# def put(self, nsname, version): +# "Create a new version for a namespace from an old version" +# try: +# return current_app.backend.ns_revert(nsname, version, get_gitactor()) +# except KeyError: +# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + + +@router.post("/db/{nsname}/q/", tags=[Tags.db]) +async def ns_query_resource_post(nsname: str, request: Request): + """Run a JSON query on the namespace and returns the results""" + req_json = await request.json() + return app.backend.ns_query(nsname, req_json) + + +# @ns_db.route("//q/") +# class NSQueryResource(Resource): +# def post(self, nsname): +# "Run a JSON query on the namespace and returns the results" +# return current_app.backend.ns_query(nsname, request.json) +# + + +@router.get("/db/{nsname}/k/", tags=[Tags.db]) +async def keys_resource_get(nsname: str): + """List all keys of a given namespace""" + return app.backend.key_list(nsname) + + +# @ns_db.route("//k/") +# class KeysResource(Resource): +# def get(self, nsname): +# "List all keys of a given namespace" +# return current_app.backend.key_list(nsname) + +@router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) +async def keys_list_versions_resource_get(nsname: str, key: str): + """Get all versions of a given key in namespace""" + return app.backend.key_list_versions(nsname, key) + + +# @ns_db.route("//k//v/") +# class KeysListVersionsResource(Resource): +# def get(self, nsname, key): +# "Get all versions of a given key in namespace" +# return current_app.backend.key_list_versions(nsname, key) +# + +@router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_get(nsname: str, key: str): + """Retrieve a given key's value from a given namespace""" + return app.backend.key_get(nsname, key) + + +@router.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_put(nsname: str, key: str, request: Request): + """Create or update the value of a key""" + # check if "reblaze/k/" exists in system/schema-validation + req_json = await request.json() + + if nsname != "system": + keyName = nsname + "/k/" + key + schemas = app.backend.key_get("system", "schema-validation") + schema = None + # find schema if exists and validate the json input + for item in schemas.items(): + if item[0] == keyName: + schema = item[1] + break + if schema: + isValid = validateDbJson(req_json, schema) + if isValid is False: + raise HTTPException(500, "schema mismatched") + return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + + +@router.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) +async def key_resource_delete(nsname: str, key: str, request: Request): + """Delete a key""" + return app.backend.key_delete(nsname, key, get_gitactor(request)) + + +# @ns_db.route("//k//") +# class KeyResource(Resource): +# # def get(self, nsname, key): +# # "Retrieve a given key's value from a given namespace" +# # return current_app.backend.key_get(nsname, key) +# +# # def put(self, nsname, key): +# # "Create or update the value of a key" +# # # check if "reblaze/k/" exists in system/schema-validation +# # if nsname != "system": +# # keyName = nsname + "/k/" + key +# # schemas = current_app.backend.key_get("system", "schema-validation") +# # schema = None +# # # find schema if exists and validate the json input +# # for item in schemas.items(): +# # if item[0] == keyName: +# # schema = item[1] +# # break +# # if schema: +# # isValid = validateDbJson(request.json, schema) +# # if isValid is False: +# # abort(500, "schema mismatched") +# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) +# +# # def delete(self, nsname, key): +# # "Delete a key" +# # return current_app.backend.key_delete(nsname, key, get_gitactor()) ############# @@ -929,82 +1619,151 @@ def delete(self, nsname, key): ############# -req_fetch_parser = reqparse.RequestParser() -req_fetch_parser.add_argument("url", location="args", help="url to retrieve") - - -@ns_tools.route("/fetch") -class FetchResource(Resource): - @ns_tools.expect(req_fetch_parser, validate=True) - def get(self): - "Fetch an URL" - args = req_fetch_parser.parse_args() +@router.get("/tools/fetch", tags=[Tags.tools]) +async def fetch_resource_get(url: str): + """Fetch an URL""" + try: + r = requests.get(url) + except Exception as e: + raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) + return r.content + + +# @ns_tools.route("/fetch") +# class FetchResource(Resource): +# @ns_tools.expect(req_fetch_parser, validate=True) +# def get(self): +# "Fetch an URL" +# args = req_fetch_parser.parse_args() +# try: +# r = requests.get(args.url) +# except Exception as e: +# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) +# return make_response(r.content) + + +@router.put("/tools/publish/{config}/", tags=[Tags.tools]) +@router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) +async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): + """Push configuration to s3 buckets""" + conf = app.backend.configs_get(config, version) + ok = True + status = [] + req_json = await request.json() + if type(req_json) is not list: + raise HTTPException(400, "body must be a list") + for bucket in buckets: + logs = [] try: - r = requests.get(args.url) + cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) except Exception as e: - abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) - return make_response(r.content) - - -@ns_tools.route("/publish//") -@ns_tools.route("/publish//v//") -class PublishResource(Resource): - @ns_tools.expect([m_bucket], validate=True) - def put(self, config, version=None): - "Push configuration to s3 buckets" - conf = current_app.backend.configs_get(config, version) - ok = True - status = [] - if type(request.json) is not list: - abort(400, "body must be a list") - for bucket in request.json: - logs = [] - try: - cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitpush/") -class GitPushResource(Resource): - @ns_tools.expect([m_giturl], validate=True) - def put(self): - "Push git configuration to remote git repositories" - ok = True - status = [] - for giturl in request.json: - try: - current_app.backend.gitpush(giturl["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": giturl["giturl"], "ok": s, "message": msg}) - return make_response({"ok": ok, "status": status}) - - -@ns_tools.route("/gitfetch/") -class GitFetchResource(Resource): - @ns_tools.expect(m_giturl, validate=True) - def put(self): - "Fetch git configuration from specified remote repository" - ok = True + ok = False + s = False + msg = repr(e) + else: + s = True + msg = "ok" + status.append( + {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + ) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/publish//") +# @ns_tools.route("/publish//v//") +# class PublishResource(Resource): +# @ns_tools.expect([m_bucket], validate=True) +# def put(self, config, version=None): +# "Push configuration to s3 buckets" +# conf = current_app.backend.configs_get(config, version) +# ok = True +# status = [] +# if type(request.json) is not list: +# abort(400, "body must be a list") +# for bucket in request.json: +# logs = [] +# try: +# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) +# except Exception as e: +# ok = False +# s = False +# msg = repr(e) +# else: +# s = True +# msg = "ok" +# status.append( +# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} +# ) +# return make_response({"ok": ok, "status": status}) + + +@router.put("/tools/gitpush/", tags=[Tags.tools]) +async def git_push_resource_put(git_urls: List[GitUrl]): + """Push git configuration to remote git repositories""" + ok = True + status = [] + for giturl in git_urls: try: - current_app.backend.gitfetch(request.json["giturl"]) + app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: - ok = False msg = repr(e) + s = False else: msg = "ok" - return make_response({"ok": ok, "status": msg}) + s = True + status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + return {"ok": ok, "status": status} + + +# @ns_tools.route("/gitpush/") +# class GitPushResource(Resource): +# @ns_tools.expect([m_giturl], validate=True) +# def put(self): +# "Push git configuration to remote git repositories" +# ok = True +# status = [] +# for giturl in request.json: +# try: +# current_app.backend.gitpush(giturl["giturl"]) +# except Exception as e: +# msg = repr(e) +# s = False +# else: +# msg = "ok" +# s = True +# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) +# return make_response({"ok": ok, "status": status}) + + +@router.put("/tools/gitfetch/", tags=[Tags.tools]) +async def git_fetch_resource_put(giturl: GitUrl): + """Fetch git configuration from specified remote repository""" + ok = True + try: + app.backend.gitfetch(dict(giturl)["giturl"]) + except Exception as e: + ok = False + msg = repr(e) + else: + msg = "ok" + return {"ok": ok, "status": msg} + + +# @ns_tools.route("/gitfetch/") +# class GitFetchResource(Resource): +# @ns_tools.expect(m_giturl, validate=True) +# def put(self): +# "Fetch git configuration from specified remote repository" +# ok = True +# try: +# current_app.backend.gitfetch(request.json["giturl"]) +# except Exception as e: +# ok = False +# msg = repr(e) +# else: +# msg = "ok" +# return make_response({"ok": ok, "status": msg}) + + +if __name__ == '__main__': + print("hi") diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index af7f2cd1a..7f84b88c8 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -33,6 +33,7 @@ "fasteners", "jsonpath-ng==1.5.3", "pydash==5.0.2", + "fastapi==0.87.0" ], classifiers=[ "Programming Language :: Python :: 3", From 9ec344e7cfe6d207ba049e985b9687e57de3f53f Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 14:54:14 +0200 Subject: [PATCH 30/55] adding request variable --- .../server/curieconf/confserver/v3/api.py | 195 +++++++++--------- 1 file changed, 97 insertions(+), 98 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 8cc9c0781..dd9ddd7bf 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -14,7 +14,6 @@ # from curieconf import utils from curieconf.utils import cloud -# from curieconf.confserver import app # TODO: TEMP DEFINITIONS @@ -107,14 +106,14 @@ class Limit(BaseModel): # securitypolicy class SecProfileMap(BaseModel): - id: str - name: str - description: str - match: str - acl_profile: str - acl_active: bool - content_filter_profile: str - content_filter_active: bool + id: str = None + name: str = None + description: Optional[str] + match: str = None + acl_profile: str = None + acl_active: bool = None + content_filter_profile: str = None + content_filter_active: bool = None limit_ids: Optional[list] @@ -141,12 +140,12 @@ class SecProfileMap(BaseModel): class SecurityPolicy(BaseModel): id: str name: str - description: str - tags: List[str] + description: Optional[str] + tags: Optional[List[str]] match: str session: anyTypeUnion session_ids: anyTypeUnion - map: List[SecProfileMap] + map: Optional[List[SecProfileMap]] # m_securitypolicy = api.model( @@ -566,10 +565,10 @@ class ConfigDocuments(BaseModel): class ConfigBlobs(BaseModel): - geolite2asn: Optional[List[Optional[BlobEntry]]] - geolite2country: Optional[List[Optional[BlobEntry]]] - geolite2city: Optional[List[Optional[BlobEntry]]] - customconf: Optional[List[Optional[BlobEntry]]] + geolite2asn: Optional[BlobEntry] + geolite2country: Optional[BlobEntry] + geolite2city: Optional[BlobEntry] + customconf: Optional[BlobEntry] # m_config_blobs = api.model( @@ -700,13 +699,13 @@ def validateDbJson(json_data, schema): def get_gitactor(request): email, username = "", "" - email_header = app.options.get("trusted_email_header", None) + email_header = request.app.options.get("trusted_email_header", None) if email_header: email = request.headers.get(email_header, "") - username_header = app.options.get("trusted_username_header", None) + username_header = request.app.options.get("trusted_username_header", None) if username_header: username = request.headers.get(username_header, "") - return app.backend.prepare_actor(username, email) + return request.app.backend.prepare_actor(username, email) base_path = Path(__file__).parent @@ -779,8 +778,8 @@ async def configs_get(request: Request): @router.post("/configs/", tags=[Tags.congifs]) async def configs_post(config: Config, request: Request): """Create a new configuration""" - data = dict(config) - return request.app.backend.configs_create(data, get_gitactor(request)) + data = await request.json() + return request.app.backend.configs_create(data=data, actor = get_gitactor(request)) # @@ -798,29 +797,29 @@ async def configs_post(config: Config, request: Request): # return current_app.backend.configs_create(data, get_gitactor()) @router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) -async def config_get(config: str): +async def config_get(config: str, request: Request): """Retrieve a complete configuration""" - return app.backend.configs_get(config) + return request.app.backend.configs_get(config) @router.post("/configs/{config}/", tags=[Tags.congifs]) async def config_post(config: str, m_config: Config, request: Request): "Create a new configuration. Configuration name in URL overrides configuration in POST data" data = dict(m_config) - return app.backend.configs_create(data, config, get_gitactor(request)) + return request.app.backend.configs_create(data, config, get_gitactor(request)) @router.put("/configs/{config}/", tags=[Tags.congifs]) async def config_put(config: str, meta: Meta, request: Request): """Update an existing configuration""" data = dict(meta) - return app.backend.configs_update(config, data, get_gitactor(request)) + return request.app.backend.configs_update(config, data, get_gitactor(request)) @router.delete("/configs/{config}/", tags=[Tags.congifs]) -async def config_delete(config: str): +async def config_delete(config: str, request: Request): """Delete a configuration""" - return app.backend.configs_delete(config) + return request.app.backend.configs_delete(config) # @ns_configs.route("//") @@ -848,10 +847,10 @@ async def config_delete(config: str): @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) -async def config_clone_post(config: str, meta: Meta): +async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" data = dict(meta) - return app.backend.configs_clone(config, data) + return request.app.backend.configs_clone(config, data) # @ns_configs.route("//clone/") @@ -864,10 +863,10 @@ async def config_clone_post(config: str, meta: Meta): # @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta): +async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): """Clone a configuration. New name is provided URL""" data = dict(meta) - return app.backend.configs_clone(config, data, new_name) + return request.app.backend.configs_clone(config, data, new_name) # @ns_configs.route("//clone//") @@ -880,9 +879,9 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta): @router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def config_list_version_get(config: str): +async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" - return app.backend.configs_list_versions(config) + return request.app.backend.configs_list_versions(config) # @ns_configs.route("//v/") @@ -894,9 +893,9 @@ async def config_list_version_get(config: str): @router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) -async def config_version_get(config: str, version: str): +async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" - return app.backend.configs_get(config, version) + return request.app.backend.configs_get(config, version) # @ns_configs.route("//v//") @@ -908,7 +907,7 @@ async def config_version_get(config: str, version: str): @router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" - return app.backend.configs_revert(config, version, get_gitactor(request)) + return request.app.backend.configs_revert(config, version, get_gitactor(request)) # @ns_configs.route("//v//revert/") @@ -924,9 +923,9 @@ async def config_revert_put(config: str, version: str, request: Request): @router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) -async def blobs_resource_get(config: str): +async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" - res = app.backend.blobs_list(config) + res = request.app.backend.blobs_list(config) return res @@ -939,15 +938,15 @@ async def blobs_resource_get(config: str): # return res @router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_resource_get(config: str, blob: str): +async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" - return app.backend.blobs_get(config, blob) + return request.app.backend.blobs_get(config, blob) @router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" - return app.backend.blobs_create( + return request.app.backend.blobs_create( config, blob, dict(blob_entry), get_gitactor(request) ) @@ -955,7 +954,7 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ @router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" - return app.backend.blobs_update( + return request.app.backend.blobs_update( config, blob, dict(blob_entry), get_gitactor(request) ) @@ -963,7 +962,7 @@ async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, reque @router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_get(config: str, blob: str, request: Request): """Delete a blob""" - return app.backend.blobs_delete(config, blob, get_gitactor(request)) + return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) # @@ -994,9 +993,9 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def blob_list_version_resource_get(config: str, blob: str): +async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" - res = app.backend.blobs_list_versions(config, blob) + res = request.app.backend.blobs_list_versions(config, blob) return res @@ -1010,9 +1009,9 @@ async def blob_list_version_resource_get(config: str, blob: str): @router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) -async def blob_version_resource_get(config: str, blob: str, version: str): +async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" - return app.backend.blobs_get(config, blob, version) + return request.app.backend.blobs_get(config, blob, version) # @ns_configs.route("//b//v//") @@ -1025,7 +1024,7 @@ async def blob_version_resource_get(config: str, blob: str, version: str): @router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): """Create a new version for a blob from an old version""" - return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) + return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) # @@ -1041,9 +1040,9 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ################# @router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) -async def document_resource(config: str): +async def document_resource(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" - res = app.backend.documents_list(config) + res = request.app.backend.documents_list(config) return res @@ -1058,11 +1057,11 @@ async def document_resource(config: str): @router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) -async def document_resource_get(config: str, document: str): +async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - res = app.backend.documents_get(config, document) + res = request.app.backend.documents_get(config, document) res = {key: res[key] for key in list(models[document].__fields__.keys())} return res @@ -1082,7 +1081,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) - res = app.backend.documents_create( + res = request.app.backend.documents_create( config, document, data, get_gitactor(request) ) return res @@ -1099,7 +1098,7 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = app.backend.documents_update( + res = request.app.backend.documents_update( config, document, data, get_gitactor(request) ) return res @@ -1110,7 +1109,7 @@ async def document_resource_delete(config: str, document: str, request: Request) """Delete/empty a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_delete(config, document, get_gitactor(request)) + res = request.app.backend.documents_delete(config, document, get_gitactor(request)) return res @@ -1163,11 +1162,11 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str): +async def document_list_version_resource_get(config: str, document: str, request: Request): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_list_versions(config, document) + res = request.app.backend.documents_list_versions(config, document) res = {key: res[key] for key in list(VersionLog.__fields__.keys())} return res @@ -1184,11 +1183,11 @@ async def document_list_version_resource_get(config: str, document: str): @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str): +async def document_version_resource_get(config: str, document: str, version: str, request: Request): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.documents_get(config, document, version) + res = request.app.backend.documents_get(config, document, version) return {key: res[key] for key in list(models[document].__fields__.keys())} @@ -1204,7 +1203,7 @@ async def document_version_resource_get(config: str, document: str, version: str @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) async def document_revert_resource_put(config: str, document: str, version: str, request: Request): """Create a new version for a document from an old version""" - return app.backend.documents_revert( + return request.app.backend.documents_revert( config, document, version, get_gitactor(request) ) @@ -1223,11 +1222,11 @@ async def document_revert_resource_put(config: str, document: str, version: str, ############### @router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_get(config: str, document: str): +async def entries_resource_get(config: str, document: str, request: Request): """Retrieve the list of entries in a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_list(config, document) + res = request.app.backend.entries_list(config, document) return res # XXX: marshal @@ -1239,7 +1238,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn isValid, err = validateJson(dict(basic_entry), document) if isValid: data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_create( + res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) return res @@ -1273,11 +1272,11 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_get(config: str, document: str, entry: str): +async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry) + res = request.app.backend.entries_get(config, document, entry) return {key: res for key in list(models[document].__fields__.keys())} @@ -1290,7 +1289,7 @@ async def entry_resource_put(config: str, document: str, entry: str, basic_entry if isValid: data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_update( + res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) ) return res @@ -1303,7 +1302,7 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: """Delete an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_delete( + res = request.app.backend.entries_delete( config, document, entry, get_gitactor(request) ) return res @@ -1344,11 +1343,11 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: @router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str): +async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_list_versions(config, document, entry) + res = request.app.backend.entries_list_versions(config, document, entry) return {key: res[key] for key in list(VersionLog.__fields__.keys())} @@ -1364,11 +1363,11 @@ async def entry_list_version_resource_get(config: str, document: str, entry: str @router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str): +async def entry_version_resource_get(config: str, document: str, entry: str, version: str, request: Request): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry, version) + res = request.app.backend.entries_get(config, document, entry, version) return {key: res[key] for key in list(models[document].__fields__.keys())} @@ -1389,9 +1388,9 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver ################ @router.get("/db/", tags=[Tags.db]) -async def db_resource_get(): +async def db_resource_get(request: Request): """Get the list of existing namespaces""" - return app.backend.ns_list() + return request.app.backend.ns_list() # @ns_db.route("/") @@ -1402,9 +1401,9 @@ async def db_resource_get(): @router.get("/db/v/", tags=[Tags.db]) -async def db_query_resource_get(): +async def db_query_resource_get(request: Request): """List all existing versions of namespaces""" - return app.backend.ns_list_versions() + return request.app.backend.ns_list_versions() # @@ -1416,10 +1415,10 @@ async def db_query_resource_get(): @router.get("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_get(nsname: str): +async def ns_resource_get(nsname: str, request: Request): """Get a complete namespace""" try: - return app.backend.ns_get(nsname, version=None) + return request.app.backend.ns_get(nsname, version=None) except KeyError: raise HTTPException(404, "namespace [%s] does not exist" % nsname) @@ -1428,7 +1427,7 @@ async def ns_resource_get(nsname: str): async def ns_resource_post(nsname: str, db: DB, request: Request): """Create a non-existing namespace from data""" try: - return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_create(nsname, dict(db), get_gitactor(request)) except Exception: raise HTTPException(409, "namespace [%s] already exists" % nsname) @@ -1436,14 +1435,14 @@ async def ns_resource_post(nsname: str, db: DB, request: Request): @router.put("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, db: DB, request: Request): """Merge data into a namespace""" - return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_update(nsname, dict(db), get_gitactor(request)) @router.delete("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, request: Request): """Delete an existing namespace""" try: - return app.backend.ns_delete(nsname, get_gitactor(request)) + return request.app.backend.ns_delete(nsname, get_gitactor(request)) except KeyError: raise HTTPException(409, "namespace [%s] does not exist" % nsname) @@ -1479,9 +1478,9 @@ async def ns_resource_put(nsname: str, request: Request): @router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) -async def ns_version_resource_get(nsname: str, version: str): +async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" - return app.backend.ns_get(nsname, version) + return request.app.backend.ns_get(nsname, version) # @ns_db.route("//v//") @@ -1495,7 +1494,7 @@ async def ns_version_resource_get(nsname: str, version: str): async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): """Create a new version for a namespace from an old version""" try: - return app.backend.ns_revert(nsname, version, get_gitactor(request)) + return request.app.backend.ns_revert(nsname, version, get_gitactor(request)) except KeyError: raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) @@ -1515,7 +1514,7 @@ async def ns_version_revert_resource_put(nsname: str, version: str, request: Req async def ns_query_resource_post(nsname: str, request: Request): """Run a JSON query on the namespace and returns the results""" req_json = await request.json() - return app.backend.ns_query(nsname, req_json) + return request.app.backend.ns_query(nsname, req_json) # @ns_db.route("//q/") @@ -1527,9 +1526,9 @@ async def ns_query_resource_post(nsname: str, request: Request): @router.get("/db/{nsname}/k/", tags=[Tags.db]) -async def keys_resource_get(nsname: str): +async def keys_resource_get(nsname: str, request: Request): """List all keys of a given namespace""" - return app.backend.key_list(nsname) + return request.app.backend.key_list(nsname) # @ns_db.route("//k/") @@ -1539,9 +1538,9 @@ async def keys_resource_get(nsname: str): # return current_app.backend.key_list(nsname) @router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) -async def keys_list_versions_resource_get(nsname: str, key: str): +async def keys_list_versions_resource_get(nsname: str, key: str, request: Request): """Get all versions of a given key in namespace""" - return app.backend.key_list_versions(nsname, key) + return request.app.backend.key_list_versions(nsname, key) # @ns_db.route("//k//v/") @@ -1552,9 +1551,9 @@ async def keys_list_versions_resource_get(nsname: str, key: str): # @router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_get(nsname: str, key: str): +async def key_resource_get(nsname: str, key: str, request: Request): """Retrieve a given key's value from a given namespace""" - return app.backend.key_get(nsname, key) + return request.app.backend.key_get(nsname, key) @router.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) @@ -1565,7 +1564,7 @@ async def key_resource_put(nsname: str, key: str, request: Request): if nsname != "system": keyName = nsname + "/k/" + key - schemas = app.backend.key_get("system", "schema-validation") + schemas = request.app.backend.key_get("system", "schema-validation") schema = None # find schema if exists and validate the json input for item in schemas.items(): @@ -1576,13 +1575,13 @@ async def key_resource_put(nsname: str, key: str, request: Request): isValid = validateDbJson(req_json, schema) if isValid is False: raise HTTPException(500, "schema mismatched") - return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) + return request.app.backend.key_set(nsname, key, req_json, get_gitactor(request)) @router.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) async def key_resource_delete(nsname: str, key: str, request: Request): """Delete a key""" - return app.backend.key_delete(nsname, key, get_gitactor(request)) + return request.app.backend.key_delete(nsname, key, get_gitactor(request)) # @ns_db.route("//k//") @@ -1646,7 +1645,7 @@ async def fetch_resource_get(url: str): @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): """Push configuration to s3 buckets""" - conf = app.backend.configs_get(config, version) + conf = request.app.backend.configs_get(config, version) ok = True status = [] req_json = await request.json() @@ -1698,13 +1697,13 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck @router.put("/tools/gitpush/", tags=[Tags.tools]) -async def git_push_resource_put(git_urls: List[GitUrl]): +async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" ok = True status = [] for giturl in git_urls: try: - app.backend.gitpush(dict(giturl)["giturl"]) + request.app.backend.gitpush(dict(giturl)["giturl"]) except Exception as e: msg = repr(e) s = False @@ -1736,11 +1735,11 @@ async def git_push_resource_put(git_urls: List[GitUrl]): @router.put("/tools/gitfetch/", tags=[Tags.tools]) -async def git_fetch_resource_put(giturl: GitUrl): +async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True try: - app.backend.gitfetch(dict(giturl)["giturl"]) + request.app.backend.gitfetch(dict(giturl)["giturl"]) except Exception as e: ok = False msg = repr(e) From 967cea7ab94c1f70d0fbfc390dbfe63f5fda1ad2 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 16:33:37 +0200 Subject: [PATCH 31/55] changing to strict --- .../server/curieconf/confserver/v3/api.py | 238 +++++++++--------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index dd9ddd7bf..0fd6f96f2 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,7 +4,7 @@ from typing import Optional, List, Union from fastapi import FastAPI, Request, HTTPException, APIRouter -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt import random # needed for generating a random number for an API import uvicorn # optional if you run it directly from terminal import jsonschema @@ -59,8 +59,8 @@ # limit class Threshold(BaseModel): - limit: int - action: str + limit: StrictInt + action: StrictStr # m_threshold = api.model( @@ -72,18 +72,18 @@ class Threshold(BaseModel): # ) class Limit(BaseModel): - id: str - name: str - description: Optional[str] - _global: bool = Field(alias="global") - active: bool - timeframe: int + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + _global: StrictBool = Field(alias="global") + active: StrictBool + timeframe: StrictInt thresholds: List[Threshold] include: typing.Any exclude: typing.Any key: anyTypeUnion pairwith: typing.Any - tags: List[str] + tags: List[StrictStr] # m_limit = api.model( @@ -106,14 +106,14 @@ class Limit(BaseModel): # securitypolicy class SecProfileMap(BaseModel): - id: str = None - name: str = None - description: Optional[str] - match: str = None - acl_profile: str = None - acl_active: bool = None - content_filter_profile: str = None - content_filter_active: bool = None + id: StrictStr = None + name: StrictStr = None + description: Optional[StrictStr] + match: StrictStr = None + acl_profile: StrictStr = None + acl_active: StrictBool = None + content_filter_profile: StrictStr = None + content_filter_active: StrictBool = None limit_ids: Optional[list] @@ -138,11 +138,11 @@ class SecProfileMap(BaseModel): # ) class SecurityPolicy(BaseModel): - id: str - name: str - description: Optional[str] - tags: Optional[List[str]] - match: str + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + tags: Optional[List[StrictStr]] + match: StrictStr session: anyTypeUnion session_ids: anyTypeUnion map: Optional[List[SecProfileMap]] @@ -165,17 +165,17 @@ class SecurityPolicy(BaseModel): # content filter rule class ContentFilterRule(BaseModel): - id: str - name: str - msg: str - operand: str - severity: int - certainity: int - category: str - subcategory: str - risk: int - tags: Optional[List[str]] - description: Optional[str] + id: StrictStr + name: StrictStr + msg: StrictStr + operand: StrictStr + severity: StrictInt + certainity: StrictInt + category: StrictStr + subcategory: StrictStr + risk: StrictInt + tags: Optional[List[StrictStr]] + description: Optional[StrictStr] # m_contentfilterrule = api.model( @@ -197,24 +197,24 @@ class ContentFilterRule(BaseModel): # content filter profile class ContentFilterProfile(BaseModel): - id: str - name: str - description: Optional[str] - ignore_alphanum: bool + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + ignore_alphanum: StrictBool args: typing.Any headers: typing.Any cookies: typing.Any path: typing.Any allsections: typing.Any decoding: typing.Any - masking_seed: str - content_type: Optional[List[str]] - active: Optional[List[str]] - report: Optional[List[str]] - ignore: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - ignore_body: bool + masking_seed: StrictStr + content_type: Optional[List[StrictStr]] + active: Optional[List[StrictStr]] + report: Optional[List[StrictStr]] + ignore: Optional[List[StrictStr]] + tags: Optional[List[StrictStr]] + action: Optional[StrictStr] + ignore_body: StrictBool # m_contentfilterprofile = api.model( @@ -243,17 +243,17 @@ class ContentFilterProfile(BaseModel): # aclprofile class ACLProfile(BaseModel): - id: str - name: str - description: Optional[str] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + allow: Optional[List[StrictStr]] + allow_bot: Optional[List[StrictStr]] + deny_bot: Optional[List[StrictStr]] + passthrough: Optional[List[StrictStr]] + deny: Optional[List[StrictStr]] + force_deny: Optional[List[StrictStr]] + tags: Optional[List[StrictStr]] + action: Optional[StrictStr] # m_aclprofile = api.model( @@ -275,14 +275,14 @@ class ACLProfile(BaseModel): # Global Filter class GlobalFilter(BaseModel): - id: str - name: str - source: str - mdate: str - description: str - active: bool + id: StrictStr + name: StrictStr + source: StrictStr + mdate: StrictStr + description: StrictStr + active: StrictBool action: typing.Any - tags: Optional[List[str]] + tags: Optional[List[StrictStr]] rule: anyTypeUnion @@ -304,16 +304,16 @@ class GlobalFilter(BaseModel): # Flow Control class FlowControl(BaseModel): - id: str - name: str - timeframe: int + id: StrictStr + name: StrictStr + timeframe: StrictInt key: List[typing.Any] sequence: List[typing.Any] - tags: Optional[List[str]] - include: Optional[List[str]] - exclude: Optional[List[str]] - description: Optional[str] - active: bool + tags: Optional[List[StrictStr]] + include: Optional[List[StrictStr]] + exclude: Optional[List[StrictStr]] + description: Optional[StrictStr] + active: StrictBool # @@ -336,12 +336,12 @@ class FlowControl(BaseModel): # Action class Action(BaseModel): - id: str - name: str - description: Optional[str] - tags: List[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] + tags: List[StrictStr] params: typing.Any - type: str + type: StrictStr # m_action = api.model( @@ -358,9 +358,9 @@ class Action(BaseModel): # Virtual Tag class VirtualTag(BaseModel): - id: str - name: str - description: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] match: List[typing.Any] @@ -377,8 +377,8 @@ class VirtualTag(BaseModel): # custom class Custom(BaseModel): - id: str - name: str + id: StrictStr + name: StrictStr # m_custom = api.model( @@ -408,33 +408,33 @@ class Custom(BaseModel): ### Other models class DocumentMask(BaseModel): - id: str - name: str - description: str + id: StrictStr + name: StrictStr + description: StrictStr map: Optional[List[SecProfileMap]] include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] - tags: Optional[List[str]] + tags: Optional[List[StrictStr]] active: Optional[List[typing.Any]] action: typing.Any sequence: Optional[List[typing.Any]] - timeframe: Optional[int] + timeframe: Optional[StrictInt] thresholds: Optional[List[Threshold]] pairwith: typing.Any - content_type: Optional[List[str]] + content_type: Optional[List[StrictStr]] params: typing.Any decoding: typing.Any - category: Optional[str] - subcategory: Optional[str] - risk: Optional[int] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - match: Optional[str] = "j" - _type: Optional[str] = Field(alias="type") + category: Optional[StrictStr] + subcategory: Optional[StrictStr] + risk: Optional[StrictInt] + allow: Optional[List[StrictStr]] + allow_bot: Optional[List[StrictStr]] + deny_bot: Optional[List[StrictStr]] + passthrough: Optional[List[StrictStr]] + deny: Optional[List[StrictStr]] + force_deny: Optional[List[StrictStr]] + match: Optional[StrictStr] = "j" + _type: Optional[StrictStr] = Field(alias="type") _star: Optional[List[typing.Any]] = Field(alias="*") @@ -473,7 +473,7 @@ class DocumentMask(BaseModel): # ) class VersionLog(BaseModel): - version: Optional[str] + version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] _star: Optional[List[typing.Any]] = Field(alias="*") @@ -490,11 +490,11 @@ class VersionLog(BaseModel): # ) class Meta(BaseModel): - id: str - description: str + id: StrictStr + description: StrictStr date: Optional[datetime.datetime] logs: Optional[List[VersionLog]] = [] - version: Optional[str] + version: Optional[StrictStr] # m_meta = api.model( @@ -509,7 +509,7 @@ class Meta(BaseModel): # ) class BlobEntry(BaseModel): - format: str + format: StrictStr blob: anyTypeUnion @@ -522,7 +522,7 @@ class BlobEntry(BaseModel): # ) class BlobListEntry(BaseModel): - name: Optional[str] + name: Optional[StrictStr] # m_blob_list_entry = api.model( @@ -533,8 +533,8 @@ class BlobListEntry(BaseModel): # ) class DocumentListEntry(BaseModel): - name: Optional[str] - entries: Optional[int] + name: Optional[StrictStr] + entries: Optional[StrictInt] # m_document_list_entry = api.model( @@ -577,10 +577,10 @@ class ConfigBlobs(BaseModel): # ) class ConfigDeleteBlobs(BaseModel): - geolite2asn: Optional[bool] - geolite2country: Optional[bool] - geolite2city: Optional[bool] - customconf: Optional[bool] + geolite2asn: Optional[StrictBool] + geolite2country: Optional[StrictBool] + geolite2city: Optional[StrictBool] + customconf: Optional[StrictBool] # m_config_delete_blobs = api.model( @@ -607,8 +607,8 @@ class Config(BaseModel): # ) class Edit(BaseModel): - path: str - value: str + path: StrictStr + value: StrictStr # m_edit = api.model( @@ -620,9 +620,9 @@ class Edit(BaseModel): # ) class BasicEntry(BaseModel): - id: str - name: str - description: Optional[str] + id: StrictStr + name: StrictStr + description: Optional[StrictStr] # m_basic_entry = api.model( @@ -637,8 +637,8 @@ class BasicEntry(BaseModel): ### Publish class Bucket(BaseModel): - name: str - url: str + name: StrictStr + url: StrictStr # m_bucket = api.model( @@ -652,7 +652,7 @@ class Bucket(BaseModel): ### Git push & pull class GitUrl(BaseModel): - giturl: str + giturl: StrictStr # m_giturl = api.model( From 7efd64c3d830ead932e34a6fe5adf8e70da02b06 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 16:55:54 +0200 Subject: [PATCH 32/55] dict to await json --- .../server/curieconf/confserver/v3/api.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 0fd6f96f2..39778bf78 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -15,7 +15,6 @@ # from curieconf import utils from curieconf.utils import cloud - # TODO: TEMP DEFINITIONS import os @@ -779,7 +778,7 @@ async def configs_get(request: Request): async def configs_post(config: Config, request: Request): """Create a new configuration""" data = await request.json() - return request.app.backend.configs_create(data=data, actor = get_gitactor(request)) + return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) # @@ -805,14 +804,14 @@ async def config_get(config: str, request: Request): @router.post("/configs/{config}/", tags=[Tags.congifs]) async def config_post(config: str, m_config: Config, request: Request): "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = dict(m_config) + data = await request.json() return request.app.backend.configs_create(data, config, get_gitactor(request)) @router.put("/configs/{config}/", tags=[Tags.congifs]) async def config_put(config: str, meta: Meta, request: Request): """Update an existing configuration""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_update(config, data, get_gitactor(request)) @@ -849,7 +848,7 @@ async def config_delete(config: str, request: Request): @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_clone(config, data) @@ -865,7 +864,7 @@ async def config_clone_post(config: str, meta: Meta, request: Request): @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): """Clone a configuration. New name is provided URL""" - data = dict(meta) + data = await request.json() return request.app.backend.configs_clone(config, data, new_name) @@ -946,16 +945,19 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" + b_entry = await request.json() return request.app.backend.blobs_create( - config, blob, dict(blob_entry), get_gitactor(request) + config, blob, b_entry, get_gitactor(request) ) @router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" + """upaate an existing blob""" + b_entry = await request.json() + return request.app.backend.blobs_update( - config, blob, dict(blob_entry), get_gitactor(request) + config, blob, b_entry, get_gitactor(request) ) @@ -1426,8 +1428,9 @@ async def ns_resource_get(nsname: str, request: Request): @router.post("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_post(nsname: str, db: DB, request: Request): """Create a non-existing namespace from data""" + _db = await request.json() try: - return request.app.backend.ns_create(nsname, dict(db), get_gitactor(request)) + return request.app.backend.ns_create(nsname, _db, get_gitactor(request)) except Exception: raise HTTPException(409, "namespace [%s] already exists" % nsname) @@ -1435,7 +1438,9 @@ async def ns_resource_post(nsname: str, db: DB, request: Request): @router.put("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_put(nsname: str, db: DB, request: Request): """Merge data into a namespace""" - return request.app.backend.ns_update(nsname, dict(db), get_gitactor(request)) + _db = await request.json() + + return request.app.backend.ns_update(nsname, _db, get_gitactor(request)) @router.delete("/db/{nsname}/", tags=[Tags.db]) @@ -1651,10 +1656,12 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck req_json = await request.json() if type(req_json) is not list: raise HTTPException(400, "body must be a list") + buck = await request.json() + for bucket in buckets: logs = [] try: - cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) + cloud.export(conf, buck["url"], prnt=lambda x: logs.append(x)) except Exception as e: ok = False s = False @@ -1663,7 +1670,7 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck s = True msg = "ok" status.append( - {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} + {"name": buck["name"], "ok": s, "logs": logs, "message": msg} ) return {"ok": ok, "status": status} @@ -1738,8 +1745,9 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True + _giturl = await request.json() try: - request.app.backend.gitfetch(dict(giturl)["giturl"]) + request.app.backend.gitfetch(_giturl["giturl"]) except Exception as e: ok = False msg = repr(e) From e48e630d048ef0bb63d3c619ad2893f4053c9b04 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Mon, 21 Nov 2022 18:34:46 +0200 Subject: [PATCH 33/55] configs actions of configs --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 39778bf78..7cdf598ae 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -877,7 +877,7 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta, request # return current_app.backend.configs_clone(config, data, new_name) -@router.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" return request.app.backend.configs_list_versions(config) @@ -891,7 +891,7 @@ async def config_list_version_get(config: str, request: Request): # return current_app.backend.configs_list_versions(config) -@router.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) +@router.get("/configs/{config}/v/{version}/", tags=[Tags.congifs]) async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" return request.app.backend.configs_get(config, version) @@ -903,7 +903,7 @@ async def config_version_get(config: str, version: str, request: Request): # "Retrieve a specific version of a configuration" # return current_app.backend.configs_get(config, version) -@router.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) +@router.get("/configs/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" return request.app.backend.configs_revert(config, version, get_gitactor(request)) From 49c5d5ccfbeda0b3c8d37118583621caa7817a1a Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 09:40:09 +0200 Subject: [PATCH 34/55] pre entries --- .../server/curieconf/confserver/v3/api.py | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 7cdf598ae..be7ed13c4 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -340,8 +340,12 @@ class Action(BaseModel): description: Optional[StrictStr] tags: List[StrictStr] params: typing.Any - type: StrictStr + _type: StrictStr = Field(alias="type") + class Config: + fields = { + "_type": "type" + } # m_action = api.model( # "Action", @@ -409,7 +413,7 @@ class Custom(BaseModel): class DocumentMask(BaseModel): id: StrictStr name: StrictStr - description: StrictStr + description: Optional[StrictStr] map: Optional[List[SecProfileMap]] include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] @@ -432,10 +436,14 @@ class DocumentMask(BaseModel): passthrough: Optional[List[StrictStr]] deny: Optional[List[StrictStr]] force_deny: Optional[List[StrictStr]] - match: Optional[StrictStr] = "j" + match: Optional[StrictStr] _type: Optional[StrictStr] = Field(alias="type") _star: Optional[List[typing.Any]] = Field(alias="*") + class Config: + fields = { + "_type": "type" + } # m_document_mask = api.model( # "Mask for document", @@ -921,7 +929,7 @@ async def config_revert_put(config: str, version: str, request: Request): ############# -@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) +@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry]) async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" res = request.app.backend.blobs_list(config) @@ -936,13 +944,13 @@ async def blobs_resource_get(config: str, request: Request): # res = current_app.backend.blobs_list(config) # return res -@router.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +@router.get("/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" return request.app.backend.blobs_get(config, blob) -@router.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +@router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): """Create a new blob""" b_entry = await request.json() @@ -951,7 +959,7 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ ) -@router.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) +@router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): """upaate an existing blob""" b_entry = await request.json() @@ -961,8 +969,8 @@ async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, reque ) -@router.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_get(config: str, blob: str, request: Request): +@router.delete("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) +async def blob_resource_delete(config: str, blob: str, request: Request): """Delete a blob""" return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) @@ -994,7 +1002,7 @@ async def blob_resource_get(config: str, blob: str, request: Request): # return current_app.backend.blobs_delete(config, blob, get_gitactor()) -@router.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" res = request.app.backend.blobs_list_versions(config, blob) @@ -1010,7 +1018,7 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques # return res -@router.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=VersionLog) async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" return request.app.backend.blobs_get(config, blob, version) @@ -1023,7 +1031,7 @@ async def blob_version_resource_get(config: str, blob: str, version: str, reques # "Retrieve the given version of a blob" # return current_app.backend.blobs_get(config, blob, version) -@router.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) +@router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): """Create a new version for a blob from an old version""" return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) @@ -1041,7 +1049,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ### DOCUMENTS ### ################# -@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) +@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) async def document_resource(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) @@ -1058,28 +1066,37 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True) async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - res = {key: res[key] for key in list(models[document].__fields__.keys())} + #res = {key: res[key] for key in list(models[document].__fields__.keys())} return res async def _filter(data, keys): - return {key: data[key] for key in keys} + filtered = {} + for key in keys: + if data.get(key, False): + filtered[key] = data[key] + return filtered + + @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): """Create a new complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: + if document not in models.keys(): + raise HTTPException(status_code=404, detail="document name is not one of the possible name in 'models' module") + + as_dict = await request.json() + print(as_dict[0]) + data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] + print(data[0]) + for entry in data: isValid, err = validateJson(dict(entry), document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) @@ -1094,8 +1111,8 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] + as_dict = await request.json() + data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] for entry in basic_entries: isValid, err = validateJson(dict(entry), document) if isValid is False: @@ -1164,12 +1181,12 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request): +async def document_list_version_resource_get(config: str, document: str, request: Request, response_model = List[VersionLog]): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_list_versions(config, document) - res = {key: res[key] for key in list(VersionLog.__fields__.keys())} + #res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] return res @@ -1190,7 +1207,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} + return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False) } for r in res] # @ns_configs.route("//d//v//") From 54050ece28083757dc33d0316b2fdb8cb83f1b27 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 12:35:48 +0200 Subject: [PATCH 35/55] config done prior to fixes --- .../server/curieconf/confserver/v3/api.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index be7ed13c4..2b8df33a7 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -74,7 +74,7 @@ class Limit(BaseModel): id: StrictStr name: StrictStr description: Optional[StrictStr] - _global: StrictBool = Field(alias="global") + global_: StrictBool = Field(alias="global") active: StrictBool timeframe: StrictInt thresholds: List[Threshold] @@ -340,7 +340,7 @@ class Action(BaseModel): description: Optional[StrictStr] tags: List[StrictStr] params: typing.Any - _type: StrictStr = Field(alias="type") + type_: StrictStr = Field(alias="type") class Config: fields = { @@ -418,7 +418,7 @@ class DocumentMask(BaseModel): include: Optional[List[typing.Any]] exclude: Optional[List[typing.Any]] tags: Optional[List[StrictStr]] - active: Optional[List[typing.Any]] + active: Optional[typing.Any] action: typing.Any sequence: Optional[List[typing.Any]] timeframe: Optional[StrictInt] @@ -437,8 +437,8 @@ class DocumentMask(BaseModel): deny: Optional[List[StrictStr]] force_deny: Optional[List[StrictStr]] match: Optional[StrictStr] - _type: Optional[StrictStr] = Field(alias="type") - _star: Optional[List[typing.Any]] = Field(alias="*") + type_: Optional[StrictStr] = Field(alias="type") + star_: Optional[List[typing.Any]] = Field(alias="*") class Config: fields = { @@ -483,7 +483,7 @@ class VersionLog(BaseModel): version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] - _star: Optional[List[typing.Any]] = Field(alias="*") + star_: Optional[List[typing.Any]] = Field(alias="*") # @@ -1066,12 +1066,13 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True, response_model_by_alias=True) async def document_resource_get(config: str, document: str, request: Request): """Get a complete document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) + print(res[0]["global"]) #res = {key: res[key] for key in list(models[document].__fields__.keys())} return res @@ -1252,11 +1253,13 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): "Create an entry in a document" + + data_json = await request.json() if document not in models: raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) + isValid, err = validateJson(data_json, document) if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + data = {key: data_json[key] for key in list(models[document].__fields__.keys())} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) @@ -1290,23 +1293,28 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn # abort(500, "schema mismatched: \n" + err) +def switch_alias(keys): + return [key[:-1] if key.endswith("_") else key for key in keys] + @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) - return {key: res for key in list(models[document].__fields__.keys())} + keys = switch_alias(list(models[document].__fields__.keys())) + return {key: res[key] for key in keys} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): """Update an entry in a document""" + data_json = await request.json() if document not in models: raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) + isValid, err = validateJson(data_json, document) if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} + data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys()))} res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1790,4 +1798,5 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): if __name__ == '__main__': - print("hi") + l = ["k_", "jjjj", "lama_"] + print(switch_alias(l)) \ No newline at end of file From bae7e0226a1dba83c1f4eb06afce4e829b19fd47 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 14:32:50 +0200 Subject: [PATCH 36/55] db done except /q/ --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 2b8df33a7..4a95b4fe3 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1507,7 +1507,7 @@ async def ns_resource_put(nsname: str, request: Request): # # abort(409, "namespace [%s] does not exist" % nsname) -@router.get("/db/{nsname}/v/{version}", tags=[Tags.db]) +@router.get("/db/{nsname}/v/{version}/", tags=[Tags.db]) async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" return request.app.backend.ns_get(nsname, version) From e76422a7bf5f738ef43f2e6ea68137218a3ea349 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 14:55:32 +0200 Subject: [PATCH 37/55] done without fixes --- .../server/curieconf/confserver/v3/api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 4a95b4fe3..f147ed653 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1678,15 +1678,14 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck conf = request.app.backend.configs_get(config, version) ok = True status = [] - req_json = await request.json() - if type(req_json) is not list: + buckets = await request.json() + if type(buckets) is not list: raise HTTPException(400, "body must be a list") - buck = await request.json() for bucket in buckets: logs = [] try: - cloud.export(conf, buck["url"], prnt=lambda x: logs.append(x)) + cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) except Exception as e: ok = False s = False @@ -1733,16 +1732,17 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" ok = True status = [] - for giturl in git_urls: + git_jsons = await request.json() + for giturl in git_jsons: try: - request.app.backend.gitpush(dict(giturl)["giturl"]) + request.app.backend.gitpush(giturl["giturl"]) except Exception as e: msg = repr(e) s = False else: msg = "ok" s = True - status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) + status.append({"url": giturl["giturl"], "ok": s, "message": msg}) return {"ok": ok, "status": status} @@ -1770,9 +1770,9 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" ok = True - _giturl = await request.json() + giturl_json = await request.json() try: - request.app.backend.gitfetch(_giturl["giturl"]) + request.app.backend.gitfetch(giturl_json["giturl"]) except Exception as e: ok = False msg = repr(e) From 9dbc778881ed0edb150a9f0c4ed896912538b6bf Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 24 Nov 2022 19:41:52 +0200 Subject: [PATCH 38/55] Version Log x-fields --- .../server/curieconf/confserver/v3/api.py | 99 +++++++++++++------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index f147ed653..0619cf082 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -18,7 +18,7 @@ # TODO: TEMP DEFINITIONS import os -router = APIRouter() +router = APIRouter(prefix="/api/v3") options = {} val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) if val: @@ -347,6 +347,7 @@ class Config: "_type": "type" } + # m_action = api.model( # "Action", # { @@ -445,6 +446,7 @@ class Config: "_type": "type" } + # m_document_mask = api.model( # "Mask for document", # { @@ -875,20 +877,35 @@ async def config_clone_name_post(config: str, new_name: str, meta: Meta, request data = await request.json() return request.app.backend.configs_clone(config, data, new_name) - -# @ns_configs.route("//clone//") -# class ConfigCloneName(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config, new_name): -# "Clone a configuration. New name is provided URL" -# data = request.json -# return current_app.backend.configs_clone(config, data, new_name) + # @ns_configs.route("//clone//") + # class ConfigCloneName(Resource): + # @ns_configs.expect(m_meta, validate=True) + # def post(self, config, new_name): + # "Clone a configuration. New name is provided URL" + # data = request.json + # return current_app.backend.configs_clone(config, data, new_name) + + +def filter_x_fields(res, x_fields): + fields = [] + if x_fields.startswith(('[', '{', '(')): + x_fields = x_fields[1:-1] + x_fields = x_fields.replace(" ", "") + fields = x_fields.split(",") + if isinstance(res, list): + return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + else: + return {field: res[field] for field in fields} -@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" - return request.app.backend.configs_list_versions(config) + res = request.app.backend.configs_list_versions(config) + if request.headers.get("X-fields", False): + res = filter_x_fields(res, request.headers["X-fields"]) + return res # @ns_configs.route("//v/") @@ -1002,10 +1019,13 @@ async def blob_resource_delete(config: str, blob: str, request: Request): # return current_app.backend.blobs_delete(config, blob, get_gitactor()) -@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) async def blob_list_version_resource_get(config: str, blob: str, request: Request): "Retrieve the list of versions of a given blob" res = request.app.backend.blobs_list_versions(config, blob) + if request.headers.get("X-fields", False): + res = filter_x_fields(res, request.headers["X-fields"]) return res @@ -1018,9 +1038,13 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques # return res -@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=VersionLog) +@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=BlobEntry) async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): """Retrieve the given version of a blob""" + + res = request.app.backend.blobs_get(config, blob, version) + if request.headers.get("X-fields", False): + res = filter_x_fields([res], request.headers["X-fields"]) return request.app.backend.blobs_get(config, blob, version) @@ -1050,7 +1074,7 @@ async def blob_revert_resource_put(config: str, blob: str, version: str, request ################# @router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) -async def document_resource(config: str, request: Request): +async def document_resource_get(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) return res @@ -1066,15 +1090,32 @@ async def document_resource(config: str, request: Request): # return res -@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=List[DocumentMask], response_model_exclude_unset=True, response_model_by_alias=True) +@router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_get(config: str, document: str, request: Request): + def filter_document_mask(res, x_fields): + fields = [] + if x_fields: + if x_fields.startswith(('[', '{', '(')): + x_fields = x_fields[1:-1] + x_fields = x_fields.replace(" ", "") + fields = x_fields.split(",") + else: + fields = switch_alias(DocumentMask.__fields__.keys()) + + print(f"fields are {fields}") + print(f"res is {res}") + + return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + """Get a complete document""" + + headers = request.headers + print(headers) if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - print(res[0]["global"]) - #res = {key: res[key] for key in list(models[document].__fields__.keys())} - return res + # res = {key: res[key] for key in list(models[document].__fields__.keys())} + return filter_document_mask(res, headers.get("x-fields", None)) async def _filter(data, keys): @@ -1085,8 +1126,6 @@ async def _filter(data, keys): return filtered - - @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): """Create a new complete document""" @@ -1098,7 +1137,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] print(data[0]) for entry in data: - isValid, err = validateJson(dict(entry), document) + isValid, err = validateJson(data, document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) res = request.app.backend.documents_create( @@ -1113,9 +1152,9 @@ async def document_resource_put(config: str, document: str, basic_entries: List[ if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) + data = [await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) for entry in as_dict] + for entry in data: + isValid, err = validateJson(entry, document) if isValid is False: raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) res = request.app.backend.documents_update( @@ -1182,12 +1221,13 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request, response_model = List[VersionLog]): +async def document_list_version_resource_get(config: str, document: str, request: Request, + response_model=List[VersionLog]): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_list_versions(config, document) - #res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] + # res_filtered = [{key: r[key] for key in list(VersionLog.__fields__.keys())} for r in res] return res @@ -1208,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False) } for r in res] + return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False)} for r in res] # @ns_configs.route("//d//v//") @@ -1296,6 +1336,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn def switch_alias(keys): return [key[:-1] if key.endswith("_") else key for key in keys] + @router.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_get(config: str, document: str, entry: str, request: Request): """Retrieve an entry from a document""" @@ -1303,7 +1344,7 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) keys = switch_alias(list(models[document].__fields__.keys())) - return {key: res[key] for key in keys} + return {key: res[key] for key in keys if res.get(key, False)} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) @@ -1799,4 +1840,4 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): if __name__ == '__main__': l = ["k_", "jjjj", "lama_"] - print(switch_alias(l)) \ No newline at end of file + print(switch_alias(l)) From 1922b662c957f1d822d1a3fcd511f4a2a2ba071e Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Sun, 27 Nov 2022 14:40:26 +0200 Subject: [PATCH 39/55] cleanup --- .../server/curieconf/confserver/v3/api.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 0619cf082..e607bc168 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1022,7 +1022,7 @@ async def blob_resource_delete(config: str, blob: str, request: Request): @router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], response_model_exclude_unset=True) async def blob_list_version_resource_get(config: str, blob: str, request: Request): - "Retrieve the list of versions of a given blob" + """Retrieve the list of versions of a given blob""" res = request.app.backend.blobs_list_versions(config, blob) if request.headers.get("X-fields", False): res = filter_x_fields(res, request.headers["X-fields"]) @@ -1410,13 +1410,16 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: # # return res -@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request): +@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs], response_model=List[VersionLog], + response_model_exclude_unset=True) +async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request, + ): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_list_versions(config, document, entry) - return {key: res[key] for key in list(VersionLog.__fields__.keys())} + return res + # return {key: res[key] for key in list(VersionLog.__fields__.keys())} # @@ -1836,8 +1839,3 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): # else: # msg = "ok" # return make_response({"ok": ok, "status": msg}) - - -if __name__ == '__main__': - l = ["k_", "jjjj", "lama_"] - print(switch_alias(l)) From 983a337dcd5b8fc342c616a69c672bec48b17ba2 Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Wed, 30 Nov 2022 10:54:04 +0200 Subject: [PATCH 40/55] before fork --- .../server/curieconf/confserver/__init__.py | 47 +++++++++++++++++-- .../confserver/backend/gitbackend.py | 2 +- .../server/curieconf/confserver/v3/api.py | 39 ++++++++------- curiefense/curieconf/server/setup.py | 5 +- curiefense/images/build-docker-images.sh | 2 +- curiefense/images/confserver/Dockerfile | 2 +- .../confserver/bootstrap/bootstrap_config.sh | 2 +- .../images/confserver/init/requirements.txt | 9 ++-- 8 files changed, 75 insertions(+), 33 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 0dbe9486f..870477c73 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -1,17 +1,50 @@ #! /usr/bin/env python3 - +import json import os + from .backend import Backends import uvicorn +import logging from curieconf.confserver.v3 import api -from prometheus_flask_exporter import PrometheusMetrics -from fastapi import FastAPI +from fastapi import FastAPI, Request, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.encoders import jsonable_encoder +from prometheus_fastapi_instrumentator import Instrumentator -app = FastAPI() +app = FastAPI(docs_url="/api/v3/") app.include_router(api.router) +@app.on_event("startup") +async def startup(): + Instrumentator().instrument(app).expose(app) + + +logging.basicConfig( + handlers=[ + logging.FileHandler("fastapi.log"), + logging.StreamHandler() + ], + level=logging.INFO, + format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', + datefmt='%H:%M:%S' +) +logger = logging.getLogger("filters-maxmind") + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + # exc_str = f'{exc}'.replace('\n', ' ').replace(' ', ' ') + # # or logger.error(f'{exc}') + # logger.error(exc_str) + # content = {'status_code': 10422, 'message': exc_str, 'data': None} + # + # return JSONResponse + return PlainTextResponse(str(exc), status_code=400) + + def drop_into_pdb(app, exception): import sys import pdb @@ -49,7 +82,7 @@ def main(args=None): options = parser.parse_args(args) - #TODO - find replacements for got_request_exception and prometheus_flask_exporter + # TODO - find replacements for got_request_exception and prometheus_flask_exporter # if options.pdb: # flask.got_request_exception.connect(drop_into_pdb) # metrics = PrometheusMetrics(app) @@ -62,3 +95,7 @@ def main(args=None): # app.run(debug=options.debug, host=options.host, port=options.port) finally: pass + + +if __name__ == '__main__': + main() diff --git a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py index d22e5a6d3..d81e1a22c 100644 --- a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py +++ b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py @@ -377,7 +377,7 @@ def configs_update(self, config, data, actor=CURIE_AUTHOR): doc = self.update_doc(doc, addd[docname]) if docname in deld: deleid = { - eid for eid, val in deld[docname].items() if val is True + entry["id"] for entry in deld[docname] } doc = [entry for entry in doc if entry["id"] not in deleid] self.add_document(docname, doc) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index e607bc168..a56925d41 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -777,7 +777,7 @@ class Tags(Enum): ### CONFIGS ### ################ -@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta]) +@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta], response_model_exclude_unset=True) async def configs_get(request: Request): """Get the detailed list of existing configurations""" res = request.app.backend.configs_list() @@ -893,9 +893,9 @@ def filter_x_fields(res, x_fields): x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") if isinstance(res, list): - return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + return [{field: r[field] for field in fields if field in r} for r in res] else: - return {field: res[field] for field in fields} + return {field: res[field] for field in fields if field in res} @router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], @@ -1105,7 +1105,7 @@ def filter_document_mask(res, x_fields): print(f"fields are {fields}") print(f"res is {res}") - return [{field: r[field] for field in fields if r.get(field, False)} for r in res] + return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" @@ -1119,11 +1119,12 @@ def filter_document_mask(res, x_fields): async def _filter(data, keys): - filtered = {} - for key in keys: - if data.get(key, False): - filtered[key] = data[key] - return filtered + # filtered = {} + # for key in keys: + # if data.get(key, False): + # filtered[key] = data[key] + # return filtered + return {key: data[key] for key in keys if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) @@ -1220,9 +1221,8 @@ async def document_resource_delete(config: str, document: str, request: Request) # # return res -@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str, request: Request, - response_model=List[VersionLog]): +@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) +async def document_list_version_resource_get(config: str, document: str, request: Request): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1248,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if r.get(key, False)} for r in res] + return [{key: r[key] for key in list(models[document].__fields__.keys()) if key in r} for r in res] # @ns_configs.route("//d//v//") @@ -1299,7 +1299,8 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in list(models[document].__fields__.keys())} + keys = list(models[document].__fields__.keys()) + data = {key: data_json[key] for key in keys if key in data_json} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) ) @@ -1344,18 +1345,19 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) keys = switch_alias(list(models[document].__fields__.keys())) - return {key: res[key] for key in keys if res.get(key, False)} + return {key: res[key] for key in keys if key in res} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): """Update an entry in a document""" data_json = await request.json() + print(f"-------------------------- active: {data_json.get('active', 'blaaaaaaa')} ------------------------------") if document not in models: raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys()))} + data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in data_json} res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1439,7 +1441,8 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} + keys = list(models[document].__fields__.keys()) + return {key: res[key] for key in keys if key in res} # @ns_configs.route( @@ -1738,7 +1741,7 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck s = True msg = "ok" status.append( - {"name": buck["name"], "ok": s, "logs": logs, "message": msg} + {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} ) return {"ok": ok, "status": status} diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index 7f84b88c8..e57e5578e 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -33,7 +33,10 @@ "fasteners", "jsonpath-ng==1.5.3", "pydash==5.0.2", - "fastapi==0.87.0" + "fastapi==0.87.0", + "prometheus-fastapi-instrumentator==5.9.1", + "pydantic==1.10.2", + "uvicorn==0.19.0" ], classifiers=[ "Programming Language :: Python :: 3", diff --git a/curiefense/images/build-docker-images.sh b/curiefense/images/build-docker-images.sh index 33179642a..38bd7e4f9 100755 --- a/curiefense/images/build-docker-images.sh +++ b/curiefense/images/build-docker-images.sh @@ -30,7 +30,7 @@ IFS=' ' read -ra RUST_DISTROS <<< "${RUST_DISTROS:-bionic focal}" if [ -n "$TESTIMG" ]; then IMAGES=("$TESTIMG") OTHER_IMAGES_DOCKER_TAG="$DOCKER_TAG" - DOCKER_TAG="test" + DOCKER_TAG="main" echo "Building only image $TESTIMG" else IMAGES=(confserver curieproxy-istio curieproxy-envoy \ diff --git a/curiefense/images/confserver/Dockerfile b/curiefense/images/confserver/Dockerfile index 8a6310d5b..14988f7b8 100644 --- a/curiefense/images/confserver/Dockerfile +++ b/curiefense/images/confserver/Dockerfile @@ -26,4 +26,4 @@ WORKDIR /app ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"] -CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini", "--callable", "app", "--module", "main"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/curiefense/images/confserver/bootstrap/bootstrap_config.sh b/curiefense/images/confserver/bootstrap/bootstrap_config.sh index 6d0a7a4a5..0574bc30d 100755 --- a/curiefense/images/confserver/bootstrap/bootstrap_config.sh +++ b/curiefense/images/confserver/bootstrap/bootstrap_config.sh @@ -4,7 +4,7 @@ set -e # to be run in an initContainer # Will deploy specified configuration as a bootstrap config, if there is no config in /cf-persistent-config/confdb -TARGETDIR="/cf-persistent-config/confdb" +TARGETDIR="/tmp/cf-persistent-config/confdb" if [ -e "$TARGETDIR" ]; then echo "Config already present in $TARGETDIR, exiting" diff --git a/curiefense/images/confserver/init/requirements.txt b/curiefense/images/confserver/init/requirements.txt index 6a4f4ea71..7ab304acf 100644 --- a/curiefense/images/confserver/init/requirements.txt +++ b/curiefense/images/confserver/init/requirements.txt @@ -15,10 +15,6 @@ configparser==5.2.0 decorator==5.1.1 fasteners==0.17.3 filelock==3.6.0 -Flask==1.1.4 -Flask-Cors==3.0.10 -Flask-PyMongo==2.3.0 -flask-restx==0.5.1 gitdb==4.0.9 GitPython==3.1.27 google-api-core==2.7.2 @@ -61,4 +57,7 @@ uWSGI==2.0.20 Werkzeug==0.16.1 xattr==0.9.9 zipp==3.8.0 -prometheus-flask-exporter==0.20.3 +fastapi==0.87.0 +prometheus-fastapi-instrumentator==5.9.1 +pydantic==1.10.2 +uvicorn==0.19.0 \ No newline at end of file From 507a1d9e8b8c916afdf5e1308928acf328d24646 Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 13:20:45 +0200 Subject: [PATCH 41/55] switch alias --- .../curieconf/server/curieconf/confserver/v3/api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index a56925d41..a7af8cd46 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -1124,7 +1124,7 @@ async def _filter(data, keys): # if data.get(key, False): # filtered[key] = data[key] # return filtered - return {key: data[key] for key in keys if key in data} + return {key: data[key] for key in switch_alias(keys) if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) @@ -1138,7 +1138,7 @@ async def document_resource_post(config: str, document: str, basic_entries: List data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] print(data[0]) for entry in data: - isValid, err = validateJson(data, document) + isValid, err = validateJson(entry, document) if isValid is False: raise HTTPException(500, "schema mismatched: \n" + err) res = request.app.backend.documents_create( @@ -1248,7 +1248,7 @@ async def document_version_resource_get(config: str, document: str, version: str if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in list(models[document].__fields__.keys()) if key in r} for r in res] + return [{key: r[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in r} for r in res] # @ns_configs.route("//d//v//") @@ -1299,7 +1299,7 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - keys = list(models[document].__fields__.keys()) + keys = switch_alias(list(models[document].__fields__.keys())) data = {key: data_json[key] for key in keys if key in data_json} res = request.app.backend.entries_create( config, document, data, get_gitactor(request) @@ -1441,7 +1441,7 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) - keys = list(models[document].__fields__.keys()) + keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @@ -1841,4 +1841,4 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): # msg = repr(e) # else: # msg = "ok" -# return make_response({"ok": ok, "status": msg}) +# return make_response({"ok": ok, "status": msg}) \ No newline at end of file From 506ee1acf7e6b0180395609ca3dfc17971eb3b60 Mon Sep 17 00:00:00 2001 From: yoavkatzman <100856091+yoavkatzman@users.noreply.github.com> Date: Wed, 30 Nov 2022 17:41:43 +0200 Subject: [PATCH 42/55] Update bootstrap_config.sh --- curiefense/images/confserver/bootstrap/bootstrap_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curiefense/images/confserver/bootstrap/bootstrap_config.sh b/curiefense/images/confserver/bootstrap/bootstrap_config.sh index 0574bc30d..6d0a7a4a5 100755 --- a/curiefense/images/confserver/bootstrap/bootstrap_config.sh +++ b/curiefense/images/confserver/bootstrap/bootstrap_config.sh @@ -4,7 +4,7 @@ set -e # to be run in an initContainer # Will deploy specified configuration as a bootstrap config, if there is no config in /cf-persistent-config/confdb -TARGETDIR="/tmp/cf-persistent-config/confdb" +TARGETDIR="/cf-persistent-config/confdb" if [ -e "$TARGETDIR" ]; then echo "Config already present in $TARGETDIR, exiting" From 1f83c077be5213d7acfbf070e12d948609f4c679 Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 16:15:14 +0200 Subject: [PATCH 43/55] black fixes --- curiefense/curieconf/server/app/main.py | 1 + .../server/curieconf/confserver/__init__.py | 15 +- .../confserver/backend/gitbackend.py | 4 +- .../server/curieconf/confserver/v3/api.py | 1045 +++-------------- curiefense/curieconf/server/setup.py | 2 +- 5 files changed, 159 insertions(+), 908 deletions(-) diff --git a/curiefense/curieconf/server/app/main.py b/curiefense/curieconf/server/app/main.py index f675059b4..776767a98 100644 --- a/curiefense/curieconf/server/app/main.py +++ b/curiefense/curieconf/server/app/main.py @@ -2,6 +2,7 @@ from curieconf.confserver.backend import Backends import os + app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") options = {} val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 870477c73..2baa918ac 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -22,14 +22,15 @@ async def startup(): Instrumentator().instrument(app).expose(app) +## Import all versions +from .v3 import api as api_v3 + + logging.basicConfig( - handlers=[ - logging.FileHandler("fastapi.log"), - logging.StreamHandler() - ], + handlers=[logging.FileHandler("fastapi.log"), logging.StreamHandler()], level=logging.INFO, - format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', - datefmt='%H:%M:%S' + format="[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s", + datefmt="%H:%M:%S", ) logger = logging.getLogger("filters-maxmind") @@ -97,5 +98,5 @@ def main(args=None): pass -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py index d81e1a22c..a6c1e3ef0 100644 --- a/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py +++ b/curiefense/curieconf/server/curieconf/confserver/backend/gitbackend.py @@ -376,9 +376,7 @@ def configs_update(self, config, data, actor=CURIE_AUTHOR): if docname in addd: doc = self.update_doc(doc, addd[docname]) if docname in deld: - deleid = { - entry["id"] for entry in deld[docname] - } + deleid = {entry["id"] for entry in deld[docname]} doc = [entry for entry in doc if entry["id"] not in deleid] self.add_document(docname, doc) self.commit("Update config [%s]%s" % (config, renamed), actor=actor) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index a7af8cd46..25e9c5a17 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -3,10 +3,8 @@ from enum import Enum from typing import Optional, List, Union -from fastapi import FastAPI, Request, HTTPException, APIRouter +from fastapi import Request, HTTPException, APIRouter from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt -import random # needed for generating a random number for an API -import uvicorn # optional if you run it directly from terminal import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type @@ -32,14 +30,6 @@ from pathlib import Path import json -# api_bp = Blueprint("api_v3", __name__) -# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - -# ns_configs = api.namespace("configs", description="Configurations") -# ns_db = api.namespace("db", description="Database") -# ns_tools = api.namespace("tools", description="Tools") - - ############## ### MODELS ### ############## @@ -51,25 +41,11 @@ anyType = ["number", "string", "boolean", "object", "array", "null"] -# class AnyType(fields.Raw): -# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] - - -# limit - class Threshold(BaseModel): limit: StrictInt action: StrictStr -# m_threshold = api.model( -# "Rate Limit Threshold", -# { -# "limit": fields.Integer(required=True), -# "action": fields.String(required=True), -# }, -# ) - class Limit(BaseModel): id: StrictStr name: StrictStr @@ -85,24 +61,6 @@ class Limit(BaseModel): tags: List[StrictStr] -# m_limit = api.model( -# "Rate Limit", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "global": fields.Boolean(required=True), -# "active": fields.Boolean(required=True), -# "timeframe": fields.Integer(required=True), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "include": fields.Raw(required=True), -# "exclude": fields.Raw(required=True), -# "key": AnyType(required=True), -# "pairwith": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# }, -# ) - # securitypolicy class SecProfileMap(BaseModel): id: StrictStr = None @@ -116,26 +74,6 @@ class SecProfileMap(BaseModel): limit_ids: Optional[list] -# m_secprofilemap = api.model( -# "Security Profile Map", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.String(required=True), -# "acl_profile": fields.String(required=True), -# "acl_active": fields.Boolean(required=True), -# "content_filter_profile": fields.String(required=True), -# "content_filter_active": fields.Boolean(required=True), -# "limit_ids": fields.List(fields.Raw()), -# }, -# ) - -# TODO = deprecated? -# m_map = api.model( -# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -# ) - class SecurityPolicy(BaseModel): id: StrictStr name: StrictStr @@ -147,22 +85,9 @@ class SecurityPolicy(BaseModel): map: Optional[List[SecProfileMap]] -# m_securitypolicy = api.model( -# "Security Policy", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String()), -# "match": fields.String(required=True), -# "session": AnyType(), -# "session_ids": AnyType(), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# }, -# ) - # content filter rule + class ContentFilterRule(BaseModel): id: StrictStr name: StrictStr @@ -177,23 +102,6 @@ class ContentFilterRule(BaseModel): description: Optional[StrictStr] -# m_contentfilterrule = api.model( -# "Content Filter Rule", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "msg": fields.String(required=True), -# "operand": fields.String(required=True), -# "severity": fields.Integer(required=True), -# "certainity": fields.Integer(required=True), -# "category": fields.String(required=True), -# "subcategory": fields.String(required=True), -# "risk": fields.Integer(required=True), -# "tags": fields.List(fields.String()), -# "description": fields.String(), -# }, -# ) - # content filter profile class ContentFilterProfile(BaseModel): id: StrictStr @@ -216,30 +124,6 @@ class ContentFilterProfile(BaseModel): ignore_body: StrictBool -# m_contentfilterprofile = api.model( -# "Content Filter Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "ignore_alphanum": fields.Boolean(required=True), -# "args": fields.Raw(required=True), -# "headers": fields.Raw(required=True), -# "cookies": fields.Raw(required=True), -# "path": fields.Raw(required=True), -# "allsections": fields.Raw(), -# "decoding": fields.Raw(required=True), -# "masking_seed": fields.String(required=True), -# "content_type": fields.List(fields.String()), -# "active": fields.List(fields.String()), -# "report": fields.List(fields.String()), -# "ignore": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# "ignore_body": fields.Boolean(required=True), -# }, -# ) - # aclprofile class ACLProfile(BaseModel): id: StrictStr @@ -255,23 +139,6 @@ class ACLProfile(BaseModel): action: Optional[StrictStr] -# m_aclprofile = api.model( -# "ACL Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# }, -# ) - # Global Filter class GlobalFilter(BaseModel): id: StrictStr @@ -285,23 +152,9 @@ class GlobalFilter(BaseModel): rule: anyTypeUnion -# m_glbalfilter = api.model( -# "Global Filter", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "source": fields.String(required=True), -# "mdate": fields.String(required=True), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# "action": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# "rule": AnyType(), -# }, -# ) - # Flow Control + class FlowControl(BaseModel): id: StrictStr name: StrictStr @@ -315,25 +168,9 @@ class FlowControl(BaseModel): active: StrictBool -# -# m_flowcontrol = api.model( -# "Flow Control", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "timeframe": fields.Integer(required=True), -# "key": fields.List(fields.Raw(required=True)), -# "sequence": fields.List(fields.Raw(required=True)), -# "tags": fields.List(fields.String()), -# "include": fields.List(fields.String()), -# "exclude": fields.List(fields.String()), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# }, -# ) - # Action + class Action(BaseModel): id: StrictStr name: StrictStr @@ -343,22 +180,8 @@ class Action(BaseModel): type_: StrictStr = Field(alias="type") class Config: - fields = { - "_type": "type" - } - + fields = {"_type": "type"} -# m_action = api.model( -# "Action", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String(required=True)), -# "params": fields.Raw(), -# "type": fields.String(required=True), -# }, -# ) # Virtual Tag class VirtualTag(BaseModel): @@ -368,32 +191,12 @@ class VirtualTag(BaseModel): match: List[typing.Any] -# -# m_virtualtag = api.model( -# "Virtual Tag", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.List(fields.Raw(required=True)), -# }, -# ) - # custom class Custom(BaseModel): id: StrictStr name: StrictStr -# m_custom = api.model( -# "Custom", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - ### mapping from doc name to model models = { @@ -442,44 +245,8 @@ class DocumentMask(BaseModel): star_: Optional[List[typing.Any]] = Field(alias="*") class Config: - fields = { - "_type": "type" - } - + fields = {"_type": "type"} -# m_document_mask = api.model( -# "Mask for document", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(required=True), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# "include": fields.Wildcard(fields.Raw()), -# "exclude": fields.Wildcard(fields.Raw()), -# "tags": fields.List(fields.String()), -# "active": fields.Wildcard(fields.Raw()), -# "action": fields.Raw(), -# "sequence": fields.List(fields.Raw()), -# "timeframe": fields.Integer(), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "pairwith": fields.Raw(), -# "content_type": fields.List(fields.String()), -# "params": fields.Raw(), -# "decoding": fields.Raw(), -# "category": fields.String(), -# "subcategory": fields.String(), -# "risk": fields.Integer(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "match": fields.String(), -# "type": fields.String(), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) class VersionLog(BaseModel): version: Optional[StrictStr] @@ -488,16 +255,6 @@ class VersionLog(BaseModel): star_: Optional[List[typing.Any]] = Field(alias="*") -# -# m_version_log = api.model( -# "Version log", -# { -# "version": fields.String(), -# "date": fields.DateTime(dt_format="iso8601"), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - class Meta(BaseModel): id: StrictStr description: StrictStr @@ -506,54 +263,20 @@ class Meta(BaseModel): version: Optional[StrictStr] -# m_meta = api.model( -# "Meta", -# { -# "id": fields.String(required=True), -# "description": fields.String(required=True), -# "date": fields.DateTime(), -# "logs": fields.List(fields.Nested(m_version_log), default=[]), -# "version": fields.String(), -# }, -# ) - class BlobEntry(BaseModel): format: StrictStr blob: anyTypeUnion -# m_blob_entry = api.model( -# "Blob Entry", -# { -# "format": fields.String(required=True), -# "blob": AnyType(), -# }, -# ) - class BlobListEntry(BaseModel): name: Optional[StrictStr] -# m_blob_list_entry = api.model( -# "Blob ListEntry", -# { -# "name": fields.String(), -# }, -# ) - class DocumentListEntry(BaseModel): name: Optional[StrictStr] entries: Optional[StrictInt] -# m_document_list_entry = api.model( -# "Document ListEntry", -# { -# "name": fields.String(), -# "entries": fields.Integer(), -# }, -# ) - class ConfigDocuments(BaseModel): ratelimits: Optional[List[models["ratelimits"]]] = [] securitypolicies: Optional[List[models["securitypolicies"]]] = [] @@ -567,12 +290,6 @@ class ConfigDocuments(BaseModel): custom: Optional[List[models["custom"]]] = [] -# m_config_documents = api.model( -# "Config Documents", -# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -# ) - - class ConfigBlobs(BaseModel): geolite2asn: Optional[BlobEntry] geolite2country: Optional[BlobEntry] @@ -580,11 +297,6 @@ class ConfigBlobs(BaseModel): customconf: Optional[BlobEntry] -# m_config_blobs = api.model( -# "Config Blobs", -# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -# ) - class ConfigDeleteBlobs(BaseModel): geolite2asn: Optional[StrictBool] geolite2country: Optional[StrictBool] @@ -592,10 +304,6 @@ class ConfigDeleteBlobs(BaseModel): customconf: Optional[StrictBool] -# m_config_delete_blobs = api.model( -# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -# ) - class Config(BaseModel): meta: Meta = {} documents: ConfigDocuments = {} @@ -604,81 +312,37 @@ class Config(BaseModel): delete_blobs: ConfigDeleteBlobs = {} -# m_config = api.model( -# "Config", -# { -# "meta": fields.Nested(m_meta, default={}), -# "documents": fields.Nested(m_config_documents, default={}), -# "blobs": fields.Nested(m_config_blobs, default={}), -# "delete_documents": fields.Nested(m_config_documents, default={}), -# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), -# }, -# ) - class Edit(BaseModel): path: StrictStr value: StrictStr -# m_edit = api.model( -# "Edit", -# { -# "path": fields.String(required=True), -# "value": fields.String(required=True), -# }, -# ) - class BasicEntry(BaseModel): id: StrictStr name: StrictStr description: Optional[StrictStr] -# m_basic_entry = api.model( -# "Basic Document Entry", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# }, -# ) - ### Publish + class Bucket(BaseModel): name: StrictStr url: StrictStr -# m_bucket = api.model( -# "Bucket", -# { -# "name": fields.String(required=True), -# "url": fields.String(required=True), -# }, -# ) - ### Git push & pull + class GitUrl(BaseModel): giturl: StrictStr -# m_giturl = api.model( -# "GitUrl", -# { -# "giturl": fields.String(required=True), -# }, -# ) - ### Db class DB(BaseModel): pass -# m_db = api.model("db", {}) - - ### Document Schema validation @@ -729,7 +393,7 @@ def get_gitactor(request): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -740,7 +404,7 @@ def get_gitactor(request): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -777,7 +441,13 @@ class Tags(Enum): ### CONFIGS ### ################ -@router.get("/configs/", tags=[Tags.congifs], response_model=List[Meta], response_model_exclude_unset=True) + +@router.get( + "/configs/", + tags=[Tags.congifs], + response_model=List[Meta], + response_model_exclude_unset=True, +) async def configs_get(request: Request): """Get the detailed list of existing configurations""" res = request.app.backend.configs_list() @@ -791,20 +461,6 @@ async def configs_post(config: Config, request: Request): return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) -# -# @ns_configs.route("/") -# class Configs(Resource): -# # @ns_configs.marshal_list_with(m_meta, skip_none=True) -# # def get(self): -# # "Get the detailed list of existing configurations" -# # return current_app.backend.configs_list() -# -# @ns_configs.expect(m_config, validate=True) -# def post(self): -# "Create a new configuration" -# data = request.json -# return current_app.backend.configs_create(data, get_gitactor()) - @router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) async def config_get(config: str, request: Request): """Retrieve a complete configuration""" @@ -831,30 +487,6 @@ async def config_delete(config: str, request: Request): return request.app.backend.configs_delete(config) -# @ns_configs.route("//") -# class Config(Resource): -# # @ns_configs.marshal_with(m_config, skip_none=True) -# # def get(self, config): -# # "Retrieve a complete configuration" -# # return current_app.backend.configs_get(config) -# -# # @ns_configs.expect(m_config, validate=True) -# # def post(self, config): -# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" -# # data = request.json -# # return current_app.backend.configs_create(data, config, get_gitactor()) -# -# # @ns_configs.expect(m_meta, validate=True) -# # def put(self, config): -# # "Update an existing configuration" -# # data = request.json -# # return current_app.backend.configs_update(config, data, get_gitactor()) -# -# def delete(self, config): -# "Delete a configuration" -# return current_app.backend.configs_delete(config) - - @router.post("/configs/{config}/clone/", tags=[Tags.congifs]) async def config_clone_post(config: str, meta: Meta, request: Request): """Clone a configuration. New name is provided in POST data""" @@ -862,33 +494,18 @@ async def config_clone_post(config: str, meta: Meta, request: Request): return request.app.backend.configs_clone(config, data) -# @ns_configs.route("//clone/") -# class ConfigClone(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config): -# "Clone a configuration. New name is provided in POST data" -# data = request.json -# return current_app.backend.configs_clone(config, data) -# - @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta, request: Request): +async def config_clone_name_post( + config: str, new_name: str, meta: Meta, request: Request +): """Clone a configuration. New name is provided URL""" data = await request.json() return request.app.backend.configs_clone(config, data, new_name) - # @ns_configs.route("//clone//") - # class ConfigCloneName(Resource): - # @ns_configs.expect(m_meta, validate=True) - # def post(self, config, new_name): - # "Clone a configuration. New name is provided URL" - # data = request.json - # return current_app.backend.configs_clone(config, data, new_name) - def filter_x_fields(res, x_fields): fields = [] - if x_fields.startswith(('[', '{', '(')): + if x_fields.startswith(("[", "{", "(")): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") @@ -898,8 +515,12 @@ def filter_x_fields(res, x_fields): return {field: res[field] for field in fields if field in res} -@router.get("/configs/{config}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) +@router.get( + "/configs/{config}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" res = request.app.backend.configs_list_versions(config) @@ -908,67 +529,44 @@ async def config_list_version_get(config: str, request: Request): return res -# @ns_configs.route("//v/") -# class ConfigListVersion(Resource): -# @ns_configs.marshal_with(m_version_log, skip_none=True) -# def get(self, config): -# "Get all versions of a given configuration" -# return current_app.backend.configs_list_versions(config) - - @router.get("/configs/{config}/v/{version}/", tags=[Tags.congifs]) async def config_version_get(config: str, version: str, request: Request): """Retrieve a specific version of a configuration""" return request.app.backend.configs_get(config, version) -# @ns_configs.route("//v//") -# class ConfigVersion(Resource): -# def get(self, config, version): -# "Retrieve a specific version of a configuration" -# return current_app.backend.configs_get(config, version) - @router.get("/configs/{config}/v/{version}/revert/", tags=[Tags.congifs]) async def config_revert_put(config: str, version: str, request: Request): """Create a new version for a configuration from an old version""" return request.app.backend.configs_revert(config, version, get_gitactor(request)) -# @ns_configs.route("//v//revert/") -# class ConfigRevert(Resource): -# def put(self, config, version): -# "Create a new version for a configuration from an old version" -# return current_app.backend.configs_revert(config, version, get_gitactor()) - - ############# ### Blobs ### ############# -@router.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry]) +@router.get( + "/configs/{config}/b/", tags=[Tags.congifs], response_model=List[BlobListEntry] +) async def blobs_resource_get(config: str, request: Request): """Retrieve the list of available blobs""" res = request.app.backend.blobs_list(config) return res -# @ns_configs.route("//b/") -# class BlobsResource(Resource): -# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of available blobs" -# res = current_app.backend.blobs_list(config) -# return res - -@router.get("/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) +@router.get( + "/configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry +) async def blob_resource_get(config: str, blob: str, request: Request): """Retrieve a blob""" return request.app.backend.blobs_get(config, blob) @router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): +async def blob_resource_post( + config: str, blob: str, blob_entry: BlobEntry, request: Request +): """Create a new blob""" b_entry = await request.json() return request.app.backend.blobs_create( @@ -977,7 +575,9 @@ async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, requ @router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): +async def blob_resource_put( + config: str, blob: str, blob_entry: BlobEntry, request: Request +): """upaate an existing blob""" b_entry = await request.json() @@ -992,35 +592,12 @@ async def blob_resource_delete(config: str, blob: str, request: Request): return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) -# -# @ns_configs.route("//b//") -# class BlobResource(Resource): -# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) -# # def get(self, config, blob): -# # "Retrieve a blob" -# # return current_app.backend.blobs_get(config, blob) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def post(self, config, blob): -# # "Create a new blob" -# # return current_app.backend.blobs_create( -# # config, blob, request.json, get_gitactor() -# # ) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def put(self, config, blob): -# # "Replace a blob with new data" -# # return current_app.backend.blobs_update( -# # config, blob, request.json, get_gitactor() -# # ) -# -# def delete(self, config, blob): -# "Delete a blob" -# return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) +@router.get( + "/configs/{config}/b/{blob}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) async def blob_list_version_resource_get(config: str, blob: str, request: Request): """Retrieve the list of versions of a given blob""" res = request.app.backend.blobs_list_versions(config, blob) @@ -1029,17 +606,14 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques return res -# @ns_configs.route("//b//v/") -# class BlobListVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob): -# "Retrieve the list of versions of a given blob" -# res = current_app.backend.blobs_list_versions(config, blob) -# return res - - -@router.get("/configs/{config}/b/{blob}/v/{version}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_version_resource_get(config: str, blob: str, version: str, request: Request): +@router.get( + "/configs/{config}/b/{blob}/v/{version}/", + tags=[Tags.congifs], + response_model=BlobEntry, +) +async def blob_version_resource_get( + config: str, blob: str, version: str, request: Request +): """Retrieve the given version of a blob""" res = request.app.backend.blobs_get(config, blob, version) @@ -1048,95 +622,73 @@ async def blob_version_resource_get(config: str, blob: str, version: str, reques return request.app.backend.blobs_get(config, blob, version) -# @ns_configs.route("//b//v//") -# class BlobVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob, version): -# "Retrieve the given version of a blob" -# return current_app.backend.blobs_get(config, blob, version) - @router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) -async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): +async def blob_revert_resource_put( + config: str, blob: str, version: str, request: Request +): """Create a new version for a blob from an old version""" - return request.app.backend.blobs_revert(config, blob, version, get_gitactor(request)) - - -# -# @ns_configs.route("//b//v//revert/") -# class BlobRevertResource(Resource): -# def put(self, config, blob, version): -# "Create a new version for a blob from an old version" -# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) + return request.app.backend.blobs_revert( + config, blob, version, get_gitactor(request) + ) ################# ### DOCUMENTS ### ################# -@router.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry]) + +@router.get( + "/configs/{config}/d/", tags=[Tags.congifs], response_model=List[DocumentListEntry] +) async def document_resource_get(config: str, request: Request): """Retrieve the list of existing documents in this configuration""" res = request.app.backend.documents_list(config) return res -# -# @ns_configs.route("//d/") -# class DocumentsResource(Resource): -# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of existing documents in this configuration" -# res = current_app.backend.documents_list(config) -# return res - - @router.get("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_get(config: str, document: str, request: Request): def filter_document_mask(res, x_fields): fields = [] if x_fields: - if x_fields.startswith(('[', '{', '(')): + if x_fields.startswith(("[", "{", "(")): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") else: fields = switch_alias(DocumentMask.__fields__.keys()) - print(f"fields are {fields}") - print(f"res is {res}") - return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" headers = request.headers - print(headers) if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) - # res = {key: res[key] for key in list(models[document].__fields__.keys())} return filter_document_mask(res, headers.get("x-fields", None)) async def _filter(data, keys): - # filtered = {} - # for key in keys: - # if data.get(key, False): - # filtered[key] = data[key] - # return filtered return {key: data[key] for key in switch_alias(keys) if key in data} @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): +async def document_resource_post( + config: str, document: str, basic_entries: List[BasicEntry], request: Request +): """Create a new complete document""" if document not in models.keys(): - raise HTTPException(status_code=404, detail="document name is not one of the possible name in 'models' module") + raise HTTPException( + status_code=404, + detail="document name is not one of the possible name in 'models' module", + ) as_dict = await request.json() - print(as_dict[0]) - data = [await _filter(dict(entry), list(models[document].__fields__.keys())) for entry in as_dict] - print(data[0]) + data = [ + await _filter(dict(entry), list(models[document].__fields__.keys())) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: @@ -1148,16 +700,23 @@ async def document_resource_post(config: str, document: str, basic_entries: List @router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): +async def document_resource_put( + config: str, document: str, basic_entries: List[BasicEntry], request: Request +): """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) for entry in as_dict] + data = [ + await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: - raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) + raise HTTPException( + 500, "schema mismatched for entry: " + str(entry) + "\n" + err + ) res = request.app.backend.documents_update( config, document, data, get_gitactor(request) ) @@ -1173,56 +732,14 @@ async def document_resource_delete(config: str, document: str, request: Request) return res -# @ns_configs.route("//d//") -# class DocumentResource(Resource): -# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) -# # def get(self, config, document): -# # "Get a complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_get(config, document) -# # return marshal(res, models[document], skip_none=True) -# # -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def post(self, config, document): -# # "Create a new complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched: \n" + err) -# # res = current_app.backend.documents_create( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def put(self, config, document): -# # "Update an existing document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) -# # res = current_app.backend.documents_update( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # def delete(self, config, document): -# # "Delete/empty a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_delete(config, document, get_gitactor()) -# # return res - - -@router.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs], response_model=List[VersionLog]) -async def document_list_version_resource_get(config: str, document: str, request: Request): +@router.get( + "/configs/{config}/d/{document}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], +) +async def document_list_version_resource_get( + config: str, document: str, request: Request +): """Retrieve the existing versions of a given document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1231,56 +748,39 @@ async def document_list_version_resource_get(config: str, document: str, request return res -# -# @ns_configs.route("//d//v/") -# class DocumentListVersionResource(Resource): -# def get(self, config, document): -# "Retrieve the existing versions of a given document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_list_versions(config, document) -# return marshal(res, m_version_log, skip_none=True) - - @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str, request: Request): +async def document_version_resource_get( + config: str, document: str, version: str, request: Request +): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [{key: r[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in r} for r in res] - + return [ + { + key: r[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in r + } + for r in res + ] -# @ns_configs.route("//d//v//") -# class DocumentVersionResource(Resource): -# def get(self, config, document, version): -# "Get a given version of a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_get(config, document, version) -# return marshal(res, models[document], skip_none=True) @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) -async def document_revert_resource_put(config: str, document: str, version: str, request: Request): +async def document_revert_resource_put( + config: str, document: str, version: str, request: Request +): """Create a new version for a document from an old version""" return request.app.backend.documents_revert( config, document, version, get_gitactor(request) ) -# @ns_configs.route("//d//v//revert/") -# class DocumentRevertResource(Resource): -# def put(self, config, document, version): -# "Create a new version for a document from an old version" -# return current_app.backend.documents_revert( -# config, document, version, get_gitactor() -# ) - - ############### ### ENTRIES ### ############### + @router.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_get(config: str, document: str, request: Request): """Retrieve the list of entries in a document""" @@ -1291,7 +791,9 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): +async def entries_resource_post( + config: str, document: str, basic_entry: BasicEntry, request: Request +): "Create an entry in a document" data_json = await request.json() @@ -1309,31 +811,6 @@ async def entries_resource_post(config: str, document: str, basic_entry: BasicEn raise HTTPException(500, "schema mismatched: \n" + err) -# @ns_configs.route("//d//e/") -# class EntriesResource(Resource): -# # def get(self, config, document): -# # "Retrieve the list of entries in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_list(config, document) -# # return res # XXX: marshal -# -# @ns_configs.expect(m_basic_entry, validate=True) -# def post(self, config, document): -# "Create an entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# isValid, err = validateJson(request.json, document) -# if isValid: -# data = marshal(request.json, models[document], skip_none=True) -# res = current_app.backend.entries_create( -# config, document, data, get_gitactor() -# ) -# return res -# else: -# abort(500, "schema mismatched: \n" + err) - - def switch_alias(keys): return [key[:-1] if key.endswith("_") else key for key in keys] @@ -1349,15 +826,20 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): +async def entry_resource_put( + config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request +): """Update an entry in a document""" data_json = await request.json() - print(f"-------------------------- active: {data_json.get('active', 'blaaaaaaa')} ------------------------------") if document not in models: raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = {key: data_json[key] for key in switch_alias(list(models[document].__fields__.keys())) if key in data_json} + data = { + key: data_json[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in data_json + } res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -1368,7 +850,9 @@ async def entry_resource_put(config: str, document: str, entry: str, basic_entry @router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): +async def entry_resource_deleye( + config: str, document: str, entry: str, request: Request +): """Delete an entry from a document""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1378,65 +862,31 @@ async def entry_resource_deleye(config: str, document: str, entry: str, request: return res -# @ns_configs.route("//d//e//") -# class EntryResource(Resource): -# # def get(self, config, document, entry): -# # "Retrieve an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_get(config, document, entry) -# # return marshal(res, models[document], skip_none=True) -# -# # @ns_configs.expect(m_basic_entry, validate=True) -# # def put(self, config, document, entry): -# # "Update an entry in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # isValid, err = validateJson(request.json, document) -# # if isValid: -# # data = marshal(request.json, models[document], skip_none=True) -# # res = current_app.backend.entries_update( -# # config, document, entry, data, get_gitactor() -# # ) -# # return res -# # else: -# # abort(500, "schema mismatched: \n" + err) -# -# # def delete(self, config, document, entry): -# # "Delete an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_delete( -# # config, document, entry, get_gitactor() -# # ) -# # return res - - -@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs], response_model=List[VersionLog], - response_model_exclude_unset=True) -async def entry_list_version_resource_get(config: str, document: str, entry: str, request: Request, - ): +@router.get( + "/configs/{config}/d/{document}/e/{entry}/v/", + tags=[Tags.congifs], + response_model=List[VersionLog], + response_model_exclude_unset=True, +) +async def entry_list_version_resource_get( + config: str, + document: str, + entry: str, + request: Request, +): """Get the list of existing versions of a given entry in a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_list_versions(config, document, entry) return res - # return {key: res[key] for key in list(VersionLog.__fields__.keys())} - - -# -# @ns_configs.route("//d//e//v/") -# class EntryListVersionResource(Resource): -# def get(self, config, document, entry): -# "Get the list of existing versions of a given entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_list_versions(config, document, entry) -# return marshal(res, m_version_log, skip_none=True) -@router.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str, request: Request): +@router.get( + "/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs] +) +async def entry_version_resource_get( + config: str, document: str, entry: str, version: str, request: Request +): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") @@ -1445,49 +895,23 @@ async def entry_version_resource_get(config: str, document: str, entry: str, ver return {key: res[key] for key in keys if key in res} -# @ns_configs.route( -# "//d//e//v//" -# ) -# class EntryVersionResource(Resource): -# def get(self, config, document, entry, version): -# "Get a given version of a document entry" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_get(config, document, entry, version) -# return marshal(res, models[document], skip_none=True) - - ################ ### Database ### ################ + @router.get("/db/", tags=[Tags.db]) async def db_resource_get(request: Request): """Get the list of existing namespaces""" return request.app.backend.ns_list() -# @ns_db.route("/") -# class DbResource(Resource): -# def get(self): -# "Get the list of existing namespaces" -# return current_app.backend.ns_list() - - @router.get("/db/v/", tags=[Tags.db]) async def db_query_resource_get(request: Request): """List all existing versions of namespaces""" return request.app.backend.ns_list_versions() -# -# @ns_db.route("/v/") -# class DbQueryResource(Resource): -# def get(self): -# "List all existing versions of namespaces" -# return current_app.backend.ns_list_versions() - - @router.get("/db/{nsname}/", tags=[Tags.db]) async def ns_resource_get(nsname: str, request: Request): """Get a complete namespace""" @@ -1524,67 +948,21 @@ async def ns_resource_put(nsname: str, request: Request): raise HTTPException(409, "namespace [%s] does not exist" % nsname) -# @ns_db.route("//") -# class NSResource(Resource): -# # def get(self, nsname): -# # "Get a complete namespace" -# # try: -# # return current_app.backend.ns_get(nsname, version=None) -# # except KeyError: -# # abort(404, "namespace [%s] does not exist" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def post(self, nsname): -# # "Create a non-existing namespace from data" -# # try: -# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) -# # except Exception: -# # abort(409, "namespace [%s] already exists" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def put(self, nsname): -# # "Merge data into a namespace" -# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) -# -# # def delete(self, nsname): -# # "Delete an existing namespace" -# # try: -# # return current_app.backend.ns_delete(nsname, get_gitactor()) -# # except KeyError: -# # abort(409, "namespace [%s] does not exist" % nsname) - - @router.get("/db/{nsname}/v/{version}/", tags=[Tags.db]) async def ns_version_resource_get(nsname: str, version: str, request: Request): """Get a given version of a namespace""" return request.app.backend.ns_get(nsname, version) -# @ns_db.route("//v//") -# class NSVersionResource(Resource): -# def get(self, nsname, version): -# "Get a given version of a namespace" -# return current_app.backend.ns_get(nsname, version) - - @router.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): """Create a new version for a namespace from an old version""" try: return request.app.backend.ns_revert(nsname, version, get_gitactor(request)) except KeyError: - raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -# -# @ns_db.route("//v//revert/") -# class NSVersionResource(Resource): -# def put(self, nsname, version): -# "Create a new version for a namespace from an old version" -# try: -# return current_app.backend.ns_revert(nsname, version, get_gitactor()) -# except KeyError: -# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) + raise HTTPException( + 404, "namespace [%s] version [%s] not found" % (nsname, version) + ) @router.post("/db/{nsname}/q/", tags=[Tags.db]) @@ -1594,39 +972,18 @@ async def ns_query_resource_post(nsname: str, request: Request): return request.app.backend.ns_query(nsname, req_json) -# @ns_db.route("//q/") -# class NSQueryResource(Resource): -# def post(self, nsname): -# "Run a JSON query on the namespace and returns the results" -# return current_app.backend.ns_query(nsname, request.json) -# - - @router.get("/db/{nsname}/k/", tags=[Tags.db]) async def keys_resource_get(nsname: str, request: Request): """List all keys of a given namespace""" return request.app.backend.key_list(nsname) -# @ns_db.route("//k/") -# class KeysResource(Resource): -# def get(self, nsname): -# "List all keys of a given namespace" -# return current_app.backend.key_list(nsname) - @router.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) async def keys_list_versions_resource_get(nsname: str, key: str, request: Request): """Get all versions of a given key in namespace""" return request.app.backend.key_list_versions(nsname, key) -# @ns_db.route("//k//v/") -# class KeysListVersionsResource(Resource): -# def get(self, nsname, key): -# "Get all versions of a given key in namespace" -# return current_app.backend.key_list_versions(nsname, key) -# - @router.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) async def key_resource_get(nsname: str, key: str, request: Request): """Retrieve a given key's value from a given namespace""" @@ -1661,35 +1018,6 @@ async def key_resource_delete(nsname: str, key: str, request: Request): return request.app.backend.key_delete(nsname, key, get_gitactor(request)) -# @ns_db.route("//k//") -# class KeyResource(Resource): -# # def get(self, nsname, key): -# # "Retrieve a given key's value from a given namespace" -# # return current_app.backend.key_get(nsname, key) -# -# # def put(self, nsname, key): -# # "Create or update the value of a key" -# # # check if "reblaze/k/" exists in system/schema-validation -# # if nsname != "system": -# # keyName = nsname + "/k/" + key -# # schemas = current_app.backend.key_get("system", "schema-validation") -# # schema = None -# # # find schema if exists and validate the json input -# # for item in schemas.items(): -# # if item[0] == keyName: -# # schema = item[1] -# # break -# # if schema: -# # isValid = validateDbJson(request.json, schema) -# # if isValid is False: -# # abort(500, "schema mismatched") -# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) -# -# # def delete(self, nsname, key): -# # "Delete a key" -# # return current_app.backend.key_delete(nsname, key, get_gitactor()) - - ############# ### Tools ### ############# @@ -1705,22 +1033,11 @@ async def fetch_resource_get(url: str): return r.content -# @ns_tools.route("/fetch") -# class FetchResource(Resource): -# @ns_tools.expect(req_fetch_parser, validate=True) -# def get(self): -# "Fetch an URL" -# args = req_fetch_parser.parse_args() -# try: -# r = requests.get(args.url) -# except Exception as e: -# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) -# return make_response(r.content) - - @router.put("/tools/publish/{config}/", tags=[Tags.tools]) @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) -async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): +async def publish_resource_put( + config: str, request: Request, buckets: List[Bucket], version: str = None +): """Push configuration to s3 buckets""" conf = request.app.backend.configs_get(config, version) ok = True @@ -1740,40 +1057,10 @@ async def publish_resource_put(config: str, request: Request, buckets: List[Buck else: s = True msg = "ok" - status.append( - {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} - ) + status.append({"name": bucket["name"], "ok": s, "logs": logs, "message": msg}) return {"ok": ok, "status": status} -# @ns_tools.route("/publish//") -# @ns_tools.route("/publish//v//") -# class PublishResource(Resource): -# @ns_tools.expect([m_bucket], validate=True) -# def put(self, config, version=None): -# "Push configuration to s3 buckets" -# conf = current_app.backend.configs_get(config, version) -# ok = True -# status = [] -# if type(request.json) is not list: -# abort(400, "body must be a list") -# for bucket in request.json: -# logs = [] -# try: -# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) -# except Exception as e: -# ok = False -# s = False -# msg = repr(e) -# else: -# s = True -# msg = "ok" -# status.append( -# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} -# ) -# return make_response({"ok": ok, "status": status}) - - @router.put("/tools/gitpush/", tags=[Tags.tools]) async def git_push_resource_put(git_urls: List[GitUrl], request: Request): """Push git configuration to remote git repositories""" @@ -1793,26 +1080,6 @@ async def git_push_resource_put(git_urls: List[GitUrl], request: Request): return {"ok": ok, "status": status} -# @ns_tools.route("/gitpush/") -# class GitPushResource(Resource): -# @ns_tools.expect([m_giturl], validate=True) -# def put(self): -# "Push git configuration to remote git repositories" -# ok = True -# status = [] -# for giturl in request.json: -# try: -# current_app.backend.gitpush(giturl["giturl"]) -# except Exception as e: -# msg = repr(e) -# s = False -# else: -# msg = "ok" -# s = True -# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) -# return make_response({"ok": ok, "status": status}) - - @router.put("/tools/gitfetch/", tags=[Tags.tools]) async def git_fetch_resource_put(giturl: GitUrl, request: Request): """Fetch git configuration from specified remote repository""" @@ -1826,19 +1093,3 @@ async def git_fetch_resource_put(giturl: GitUrl, request: Request): else: msg = "ok" return {"ok": ok, "status": msg} - - -# @ns_tools.route("/gitfetch/") -# class GitFetchResource(Resource): -# @ns_tools.expect(m_giturl, validate=True) -# def put(self): -# "Fetch git configuration from specified remote repository" -# ok = True -# try: -# current_app.backend.gitfetch(request.json["giturl"]) -# except Exception as e: -# ok = False -# msg = repr(e) -# else: -# msg = "ok" -# return make_response({"ok": ok, "status": msg}) \ No newline at end of file diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index e57e5578e..d5a5fee86 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -36,7 +36,7 @@ "fastapi==0.87.0", "prometheus-fastapi-instrumentator==5.9.1", "pydantic==1.10.2", - "uvicorn==0.19.0" + "uvicorn==0.19.0", ], classifiers=[ "Programming Language :: Python :: 3", From e1b149e401fe39d21d4e63bc456a8e7c02c46fe1 Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 1 Dec 2022 16:17:37 +0200 Subject: [PATCH 44/55] delete temp api file --- .../curieconf/confserver/v3/fast_api.py | 1773 ----------------- 1 file changed, 1773 deletions(-) delete mode 100644 curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py b/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py deleted file mode 100644 index 076af7992..000000000 --- a/curiefense/curieconf/server/curieconf/confserver/v3/fast_api.py +++ /dev/null @@ -1,1773 +0,0 @@ -import datetime -import typing -from enum import Enum -from typing import Optional, List, Union - -from fastapi import FastAPI, Request, HTTPException -from pydantic import BaseModel, Field, validator -import random # needed for generating a random number for an API -import uvicorn # optional if you run it directly from terminal -import jsonschema - -# monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type -jsonschema.Draft4Validator = jsonschema.Draft3Validator - -# from curieconf import utils -from curieconf.utils import cloud -# from curieconf.confserver import app - - -# TODO: TEMP DEFINITIONS -import os - -app = FastAPI() -app.backend = Backends.get_backend(app, "git:///cf-persistent-config/confdb") -options = {} -val = os.environ.get("CURIECONF_TRUSTED_USERNAME_HEADER", None) -if val: - options["trusted_username_header"] = val -val = os.environ.get("CURIECONF_TRUSTED_EMAIL_HEADER", None) -if val: - options["trusted_email_header"] = val - -import requests -from jsonschema import validate -from pathlib import Path -import json - -# api_bp = Blueprint("api_v3", __name__) -# api = Api(api_bp, version="3.0", title="Curiefense configuration API server v3.0") - -# ns_configs = api.namespace("configs", description="Configurations") -# ns_db = api.namespace("db", description="Database") -# ns_tools = api.namespace("tools", description="Tools") - - -############## -### MODELS ### -############## - - -### Models for documents -anyTypeUnion = Union[int, float, bool, object, list, None] -anyOp = Optional[object] -anyType = ["number", "string", "boolean", "object", "array", "null"] - - -# class AnyType(fields.Raw): -# __schema_type__ = ["number", "string", "boolean", "object", "array", "null"] - - -# limit - -class Threshold(BaseModel): - limit: int - action: str - - -# m_threshold = api.model( -# "Rate Limit Threshold", -# { -# "limit": fields.Integer(required=True), -# "action": fields.String(required=True), -# }, -# ) - -class Limit(BaseModel): - id: str - name: str - description: Optional[str] - _global: bool = Field(alias="global") - active: bool - timeframe: int - thresholds: List[Threshold] - include: typing.Any - exclude: typing.Any - key: anyTypeUnion - pairwith: typing.Any - tags: List[str] - - -# m_limit = api.model( -# "Rate Limit", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "global": fields.Boolean(required=True), -# "active": fields.Boolean(required=True), -# "timeframe": fields.Integer(required=True), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "include": fields.Raw(required=True), -# "exclude": fields.Raw(required=True), -# "key": AnyType(required=True), -# "pairwith": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# }, -# ) - -# securitypolicy -class SecProfileMap(BaseModel): - id: str - name: str - description: str - match: str - acl_profile: str - acl_active: bool - content_filter_profile: str - content_filter_active: bool - limit_ids: Optional[list] - - -# m_secprofilemap = api.model( -# "Security Profile Map", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.String(required=True), -# "acl_profile": fields.String(required=True), -# "acl_active": fields.Boolean(required=True), -# "content_filter_profile": fields.String(required=True), -# "content_filter_active": fields.Boolean(required=True), -# "limit_ids": fields.List(fields.Raw()), -# }, -# ) - -# TODO = deprecated? -# m_map = api.model( -# "Security Profile Map", {"*": fields.Wildcard(fields.Nested(m_secprofilemap))} -# ) - -class SecurityPolicy(BaseModel): - id: str - name: str - description: str - tags: List[str] - match: str - session: anyTypeUnion - session_ids: anyTypeUnion - map: List[SecProfileMap] - - -# m_securitypolicy = api.model( -# "Security Policy", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String()), -# "match": fields.String(required=True), -# "session": AnyType(), -# "session_ids": AnyType(), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# }, -# ) - -# content filter rule - -class ContentFilterRule(BaseModel): - id: str - name: str - msg: str - operand: str - severity: int - certainity: int - category: str - subcategory: str - risk: int - tags: Optional[List[str]] - description: Optional[str] - - -# m_contentfilterrule = api.model( -# "Content Filter Rule", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "msg": fields.String(required=True), -# "operand": fields.String(required=True), -# "severity": fields.Integer(required=True), -# "certainity": fields.Integer(required=True), -# "category": fields.String(required=True), -# "subcategory": fields.String(required=True), -# "risk": fields.Integer(required=True), -# "tags": fields.List(fields.String()), -# "description": fields.String(), -# }, -# ) - -# content filter profile -class ContentFilterProfile(BaseModel): - id: str - name: str - description: Optional[str] - ignore_alphanum: bool - args: typing.Any - headers: typing.Any - cookies: typing.Any - path: typing.Any - allsections: typing.Any - decoding: typing.Any - masking_seed: str - content_type: Optional[List[str]] - active: Optional[List[str]] - report: Optional[List[str]] - ignore: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - ignore_body: bool - - -# m_contentfilterprofile = api.model( -# "Content Filter Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "ignore_alphanum": fields.Boolean(required=True), -# "args": fields.Raw(required=True), -# "headers": fields.Raw(required=True), -# "cookies": fields.Raw(required=True), -# "path": fields.Raw(required=True), -# "allsections": fields.Raw(), -# "decoding": fields.Raw(required=True), -# "masking_seed": fields.String(required=True), -# "content_type": fields.List(fields.String()), -# "active": fields.List(fields.String()), -# "report": fields.List(fields.String()), -# "ignore": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# "ignore_body": fields.Boolean(required=True), -# }, -# ) - -# aclprofile -class ACLProfile(BaseModel): - id: str - name: str - description: Optional[str] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - tags: Optional[List[str]] - action: Optional[str] - - -# m_aclprofile = api.model( -# "ACL Profile", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "tags": fields.List(fields.String()), -# "action": fields.String(), -# }, -# ) - -# Global Filter -class GlobalFilter(BaseModel): - id: str - name: str - source: str - mdate: str - description: str - active: bool - action: typing.Any - tags: Optional[List[str]] - rule: anyTypeUnion - - -# m_glbalfilter = api.model( -# "Global Filter", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "source": fields.String(required=True), -# "mdate": fields.String(required=True), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# "action": fields.Raw(required=True), -# "tags": fields.List(fields.String()), -# "rule": AnyType(), -# }, -# ) - -# Flow Control - -class FlowControl(BaseModel): - id: str - name: str - timeframe: int - key: List[typing.Any] - sequence: List[typing.Any] - tags: Optional[List[str]] - include: Optional[List[str]] - exclude: Optional[List[str]] - description: Optional[str] - active: bool - - -# -# m_flowcontrol = api.model( -# "Flow Control", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "timeframe": fields.Integer(required=True), -# "key": fields.List(fields.Raw(required=True)), -# "sequence": fields.List(fields.Raw(required=True)), -# "tags": fields.List(fields.String()), -# "include": fields.List(fields.String()), -# "exclude": fields.List(fields.String()), -# "description": fields.String(), -# "active": fields.Boolean(required=True), -# }, -# ) - -# Action - -class Action(BaseModel): - id: str - name: str - description: Optional[str] - tags: List[str] - params: typing.Any - type: str - - -# m_action = api.model( -# "Action", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "tags": fields.List(fields.String(required=True)), -# "params": fields.Raw(), -# "type": fields.String(required=True), -# }, -# ) - -# Virtual Tag -class VirtualTag(BaseModel): - id: str - name: str - description: Optional[str] - match: List[typing.Any] - - -# -# m_virtualtag = api.model( -# "Virtual Tag", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# "match": fields.List(fields.Raw(required=True)), -# }, -# ) - -# custom -class Custom(BaseModel): - id: str - name: str - - -# m_custom = api.model( -# "Custom", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -### mapping from doc name to model - -models = { - "ratelimits": Limit, - "securitypolicies": SecurityPolicy, - "contentfilterrules": ContentFilterRule, - "contentfilterprofiles": ContentFilterProfile, - "aclprofiles": ACLProfile, - "globalfilters": GlobalFilter, - "flowcontrol": FlowControl, - "actions": Action, - "virtualtags": Custom, - "custom": Custom, -} - - -### Other models -class DocumentMask(BaseModel): - id: str - name: str - description: str - map: Optional[List[SecProfileMap]] - include: Optional[List[typing.Any]] - exclude: Optional[List[typing.Any]] - tags: Optional[List[str]] - active: Optional[List[typing.Any]] - action: typing.Any - sequence: Optional[List[typing.Any]] - timeframe: Optional[int] - thresholds: Optional[List[Threshold]] - pairwith: typing.Any - content_type: Optional[List[str]] - params: typing.Any - decoding: typing.Any - category: Optional[str] - subcategory: Optional[str] - risk: Optional[int] - allow: Optional[List[str]] - allow_bot: Optional[List[str]] - deny_bot: Optional[List[str]] - passthrough: Optional[List[str]] - deny: Optional[List[str]] - force_deny: Optional[List[str]] - match: Optional[str] = "j" - _type: Optional[str] = Field(alias="type") - _star: Optional[List[typing.Any]] = Field(alias="*") - - -# m_document_mask = api.model( -# "Mask for document", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(required=True), -# "map": fields.List(fields.Nested(m_secprofilemap)), -# "include": fields.Wildcard(fields.Raw()), -# "exclude": fields.Wildcard(fields.Raw()), -# "tags": fields.List(fields.String()), -# "active": fields.Wildcard(fields.Raw()), -# "action": fields.Raw(), -# "sequence": fields.List(fields.Raw()), -# "timeframe": fields.Integer(), -# "thresholds": fields.List(fields.Nested(m_threshold)), -# "pairwith": fields.Raw(), -# "content_type": fields.List(fields.String()), -# "params": fields.Raw(), -# "decoding": fields.Raw(), -# "category": fields.String(), -# "subcategory": fields.String(), -# "risk": fields.Integer(), -# "allow": fields.List(fields.String()), -# "allow_bot": fields.List(fields.String()), -# "deny_bot": fields.List(fields.String()), -# "passthrough": fields.List(fields.String()), -# "deny": fields.List(fields.String()), -# "force_deny": fields.List(fields.String()), -# "match": fields.String(), -# "type": fields.String(), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -class VersionLog(BaseModel): - version: Optional[str] - # TODO - dt_format="iso8601" - date: Optional[datetime.datetime] - _star: Optional[List[typing.Any]] = Field(alias="*") - - -# -# m_version_log = api.model( -# "Version log", -# { -# "version": fields.String(), -# "date": fields.DateTime(dt_format="iso8601"), -# "*": fields.Wildcard(fields.Raw()), -# }, -# ) - -class Meta(BaseModel): - id: str - description: str - date: Optional[datetime.datetime] - logs: Optional[List[VersionLog]] = [] - version: Optional[str] - - -# m_meta = api.model( -# "Meta", -# { -# "id": fields.String(required=True), -# "description": fields.String(required=True), -# "date": fields.DateTime(), -# "logs": fields.List(fields.Nested(m_version_log), default=[]), -# "version": fields.String(), -# }, -# ) - -class BlobEntry(BaseModel): - format: str - blob: anyTypeUnion - - -# m_blob_entry = api.model( -# "Blob Entry", -# { -# "format": fields.String(required=True), -# "blob": AnyType(), -# }, -# ) - -class BlobListEntry(BaseModel): - name: Optional[str] - - -# m_blob_list_entry = api.model( -# "Blob ListEntry", -# { -# "name": fields.String(), -# }, -# ) - -class DocumentListEntry(BaseModel): - name: Optional[str] - entries: Optional[int] - - -# m_document_list_entry = api.model( -# "Document ListEntry", -# { -# "name": fields.String(), -# "entries": fields.Integer(), -# }, -# ) - -class ConfigDocuments(BaseModel): - ratelimits: Optional[List[models["ratelimits"]]] = [] - securitypolicies: Optional[List[models["securitypolicies"]]] = [] - contentfilterrules: Optional[List[models["contentfilterrules"]]] = [] - contentfilterprofiles: Optional[List[models["contentfilterprofiles"]]] = [] - aclprofiles: Optional[List[models["aclprofiles"]]] = [] - globalfilters: Optional[List[models["globalfilters"]]] = [] - flowcontrol: Optional[List[models["flowcontrol"]]] = [] - actions: Optional[List[models["actions"]]] = [] - virtualtags: Optional[List[models["virtualtags"]]] = [] - custom: Optional[List[models["custom"]]] = [] - - -# m_config_documents = api.model( -# "Config Documents", -# {x: fields.List(fields.Nested(models[x], default=[])) for x in models}, -# ) - - -class ConfigBlobs(BaseModel): - geolite2asn: Optional[List[Optional[BlobEntry]]] - geolite2country: Optional[List[Optional[BlobEntry]]] - geolite2city: Optional[List[Optional[BlobEntry]]] - customconf: Optional[List[Optional[BlobEntry]]] - - -# m_config_blobs = api.model( -# "Config Blobs", -# {x: fields.Nested(m_blob_entry, default={}) for x in utils.BLOBS_PATH}, -# ) - -class ConfigDeleteBlobs(BaseModel): - geolite2asn: Optional[bool] - geolite2country: Optional[bool] - geolite2city: Optional[bool] - customconf: Optional[bool] - - -# m_config_delete_blobs = api.model( -# "Config Delete Blobs", {x: fields.Boolean() for x in utils.BLOBS_PATH} -# ) - -class Config(BaseModel): - meta: Meta = {} - documents: ConfigDocuments = {} - blobs: ConfigBlobs = {} - delete_documents: ConfigDocuments = {} - delete_blobs: ConfigDeleteBlobs = {} - - -# m_config = api.model( -# "Config", -# { -# "meta": fields.Nested(m_meta, default={}), -# "documents": fields.Nested(m_config_documents, default={}), -# "blobs": fields.Nested(m_config_blobs, default={}), -# "delete_documents": fields.Nested(m_config_documents, default={}), -# "delete_blobs": fields.Nested(m_config_delete_blobs, default={}), -# }, -# ) - -class Edit(BaseModel): - path: str - value: str - - -# m_edit = api.model( -# "Edit", -# { -# "path": fields.String(required=True), -# "value": fields.String(required=True), -# }, -# ) - -class BasicEntry(BaseModel): - id: str - name: str - description: Optional[str] - - -# m_basic_entry = api.model( -# "Basic Document Entry", -# { -# "id": fields.String(required=True), -# "name": fields.String(required=True), -# "description": fields.String(), -# }, -# ) - -### Publish - -class Bucket(BaseModel): - name: str - url: str - - -# m_bucket = api.model( -# "Bucket", -# { -# "name": fields.String(required=True), -# "url": fields.String(required=True), -# }, -# ) - -### Git push & pull - -class GitUrl(BaseModel): - giturl: str - - -# m_giturl = api.model( -# "GitUrl", -# { -# "giturl": fields.String(required=True), -# }, -# ) - -### Db -class DB(BaseModel): - pass - - -# m_db = api.model("db", {}) - - -### Document Schema validation - - -def validateJson(json_data, schema_type): - try: - validate(instance=json_data, schema=schema_type_map[schema_type]) - except jsonschema.exceptions.ValidationError as err: - print(str(err)) - return False, str(err) - return True, "" - - -### DB Schema validation - - -def validateDbJson(json_data, schema): - try: - validate(instance=json_data, schema=schema) - except jsonschema.exceptions.ValidationError as err: - print(str(err)) - return False - return True - - -### Set git actor according to config & defined HTTP headers - - -def get_gitactor(request): - email, username = "", "" - email_header = app.options.get("trusted_email_header", None) - if email_header: - email = request.headers.get(email_header, "") - username_header = app.options.get("trusted_username_header", None) - if username_header: - username = request.headers.get(username_header, "") - return app.backend.prepare_actor(username, email) - - -base_path = Path(__file__).parent -# base_path = "/etc/curiefense/json/" -acl_profile_file_path = (base_path / "./json/acl-profile.schema").resolve() -with open(acl_profile_file_path) as json_file: - acl_profile_schema = json.load(json_file) -ratelimits_file_path = (base_path / "./json/rate-limits.schema").resolve() -with open(ratelimits_file_path) as json_file: - ratelimits_schema = json.load(json_file) -securitypolicies_file_path = (base_path / "./json/security-policies.schema").resolve() -with open(securitypolicies_file_path) as json_file: - securitypolicies_schema = json.load(json_file) -content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" -).resolve() -with open(content_filter_profile_file_path) as json_file: - content_filter_profile_schema = json.load(json_file) -globalfilters_file_path = (base_path / "./json/global-filters.schema").resolve() -with open(globalfilters_file_path) as json_file: - globalfilters_schema = json.load(json_file) -flowcontrol_file_path = (base_path / "./json/flow-control.schema").resolve() -with open(flowcontrol_file_path) as json_file: - flowcontrol_schema = json.load(json_file) -content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" -).resolve() -with open(content_filter_rule_file_path) as json_file: - content_filter_rule_schema = json.load(json_file) -action_file_path = (base_path / "./json/action.schema").resolve() -with open(action_file_path) as json_file: - action_schema = json.load(json_file) -virtualtag_file_path = (base_path / "./json/virtual-tags.schema").resolve() -with open(virtualtag_file_path) as json_file: - virtual_tags_schema = json.load(json_file) -custom_file_path = (base_path / "./json/custom.schema").resolve() -with open(custom_file_path) as json_file: - custom_schema = json.load(json_file) -schema_type_map = { - "ratelimits": ratelimits_schema, - "securitypolicies": securitypolicies_schema, - "contentfilterprofiles": content_filter_profile_schema, - "aclprofiles": acl_profile_schema, - "globalfilters": globalfilters_schema, - "flowcontrol": flowcontrol_schema, - "contentfilterrules": content_filter_rule_schema, - "actions": action_schema, - "virtualtags": virtual_tags_schema, - "custom": custom_schema, -} - - -class Tags(Enum): - congifs = "configs" - db = "db" - tools = "tools" - - -################ -### CONFIGS ### -################ - -@app.get("/configs/", tags=[Tags.congifs], response_model=Meta) -async def configs_get(): - """Get the detailed list of existing configurations""" - return app.backend.configs_list() - - -@app.post("/configs/", tags=[Tags.congifs]) -async def configs_post(config: Config, request: Request): - """Create a new configuration""" - data = dict(config) - return app.backend.configs_create(data, get_gitactor(request)) - - -# -# @ns_configs.route("/") -# class Configs(Resource): -# # @ns_configs.marshal_list_with(m_meta, skip_none=True) -# # def get(self): -# # "Get the detailed list of existing configurations" -# # return current_app.backend.configs_list() -# -# @ns_configs.expect(m_config, validate=True) -# def post(self): -# "Create a new configuration" -# data = request.json -# return current_app.backend.configs_create(data, get_gitactor()) - -@app.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) -async def config_get(config: str): - """Retrieve a complete configuration""" - return app.backend.configs_get(config) - - -@app.post("/configs/{config}/", tags=[Tags.congifs]) -async def config_post(config: str, m_config: Config, request: Request): - "Create a new configuration. Configuration name in URL overrides configuration in POST data" - data = dict(m_config) - return app.backend.configs_create(data, config, get_gitactor(request)) - - -@app.put("/configs/{config}/", tags=[Tags.congifs]) -async def config_put(config: str, meta: Meta, request: Request): - """Update an existing configuration""" - data = dict(meta) - return app.backend.configs_update(config, data, get_gitactor(request)) - - -@app.delete("/configs/{config}/", tags=[Tags.congifs]) -async def config_delete(config: str): - """Delete a configuration""" - return app.backend.configs_delete(config) - - -# @ns_configs.route("//") -# class Config(Resource): -# # @ns_configs.marshal_with(m_config, skip_none=True) -# # def get(self, config): -# # "Retrieve a complete configuration" -# # return current_app.backend.configs_get(config) -# -# # @ns_configs.expect(m_config, validate=True) -# # def post(self, config): -# # "Create a new configuration. Configuration name in URL overrides configuration in POST data" -# # data = request.json -# # return current_app.backend.configs_create(data, config, get_gitactor()) -# -# # @ns_configs.expect(m_meta, validate=True) -# # def put(self, config): -# # "Update an existing configuration" -# # data = request.json -# # return current_app.backend.configs_update(config, data, get_gitactor()) -# -# def delete(self, config): -# "Delete a configuration" -# return current_app.backend.configs_delete(config) - - -@app.post("/configs/{config}/clone/", tags=[Tags.congifs]) -async def config_clone_post(config: str, meta: Meta): - """Clone a configuration. New name is provided in POST data""" - data = dict(meta) - return app.backend.configs_clone(config, data) - - -# @ns_configs.route("//clone/") -# class ConfigClone(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config): -# "Clone a configuration. New name is provided in POST data" -# data = request.json -# return current_app.backend.configs_clone(config, data) -# - -@app.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) -async def config_clone_name_post(config: str, new_name: str, meta: Meta): - """Clone a configuration. New name is provided URL""" - data = dict(meta) - return app.backend.configs_clone(config, data, new_name) - - -# @ns_configs.route("//clone//") -# class ConfigCloneName(Resource): -# @ns_configs.expect(m_meta, validate=True) -# def post(self, config, new_name): -# "Clone a configuration. New name is provided URL" -# data = request.json -# return current_app.backend.configs_clone(config, data, new_name) - - -@app.get("configs/{config}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def config_list_version_get(config: str): - """Get all versions of a given configuration""" - return app.backend.configs_list_versions(config) - - -# @ns_configs.route("//v/") -# class ConfigListVersion(Resource): -# @ns_configs.marshal_with(m_version_log, skip_none=True) -# def get(self, config): -# "Get all versions of a given configuration" -# return current_app.backend.configs_list_versions(config) - - -@app.get("configs/{config}/v/{version}/", tags=[Tags.congifs]) -async def config_version_get(config: str, version: str): - """Retrieve a specific version of a configuration""" - return app.backend.configs_get(config, version) - - -# @ns_configs.route("//v//") -# class ConfigVersion(Resource): -# def get(self, config, version): -# "Retrieve a specific version of a configuration" -# return current_app.backend.configs_get(config, version) - -@app.get("/{config}/v/{version}/revert/", tags=[Tags.congifs]) -async def config_revert_put(config: str, version: str, request: Request): - """Create a new version for a configuration from an old version""" - return app.backend.configs_revert(config, version, get_gitactor(request)) - - -# @ns_configs.route("//v//revert/") -# class ConfigRevert(Resource): -# def put(self, config, version): -# "Create a new version for a configuration from an old version" -# return current_app.backend.configs_revert(config, version, get_gitactor()) - - -############# -### Blobs ### -############# - - -@app.get("/configs/{config}/b/", tags=[Tags.congifs], response_model=BlobListEntry) -async def blobs_resource_get(config: str): - """Retrieve the list of available blobs""" - res = app.backend.blobs_list(config) - return res - - -# @ns_configs.route("//b/") -# class BlobsResource(Resource): -# @ns_configs.marshal_with(m_blob_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of available blobs" -# res = current_app.backend.blobs_list(config) -# return res - -@app.get("configs/{config}/b/{blob}/", tags=[Tags.congifs], response_model=BlobEntry) -async def blob_resource_get(config: str, blob: str): - """Retrieve a blob""" - return app.backend.blobs_get(config, blob) - - -@app.post("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_post(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" - return app.backend.blobs_create( - config, blob, dict(blob_entry), get_gitactor(request) - ) - - -@app.put("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_put(config: str, blob: str, blob_entry: BlobEntry, request: Request): - """Create a new blob""" - return app.backend.blobs_update( - config, blob, dict(blob_entry), get_gitactor(request) - ) - - -@app.delete("configs/{config}/b/{blob}/", tags=[Tags.congifs]) -async def blob_resource_get(config: str, blob: str, request: Request): - """Delete a blob""" - return app.backend.blobs_delete(config, blob, get_gitactor(request)) - - -# -# @ns_configs.route("//b//") -# class BlobResource(Resource): -# # @ns_configs.marshal_with(m_blob_entry, skip_none=True) -# # def get(self, config, blob): -# # "Retrieve a blob" -# # return current_app.backend.blobs_get(config, blob) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def post(self, config, blob): -# # "Create a new blob" -# # return current_app.backend.blobs_create( -# # config, blob, request.json, get_gitactor() -# # ) -# -# # @ns_configs.expect(m_blob_entry, validate=True) -# # def put(self, config, blob): -# # "Replace a blob with new data" -# # return current_app.backend.blobs_update( -# # config, blob, request.json, get_gitactor() -# # ) -# -# def delete(self, config, blob): -# "Delete a blob" -# return current_app.backend.blobs_delete(config, blob, get_gitactor()) - - -@app.get("configs/{config}/b/{blob}/v/", tags=[Tags.congifs], response_model=VersionLog) -async def blob_list_version_resource_get(config: str, blob: str): - "Retrieve the list of versions of a given blob" - res = app.backend.blobs_list_versions(config, blob) - return res - - -# @ns_configs.route("//b//v/") -# class BlobListVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob): -# "Retrieve the list of versions of a given blob" -# res = current_app.backend.blobs_list_versions(config, blob) -# return res - - -@app.get("configs/{config}/b/{blob}/v/{version}", tags=[Tags.congifs], response_model=VersionLog) -async def blob_version_resource_get(config: str, blob: str, version: str): - """Retrieve the given version of a blob""" - return app.backend.blobs_get(config, blob, version) - - -# @ns_configs.route("//b//v//") -# class BlobVersionResource(Resource): -# @ns_configs.marshal_list_with(m_version_log, skip_none=True) -# def get(self, config, blob, version): -# "Retrieve the given version of a blob" -# return current_app.backend.blobs_get(config, blob, version) - -@app.put("configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) -async def blob_revert_resource_put(config: str, blob: str, version: str, request: Request): - """Create a new version for a blob from an old version""" - return app.backend.blobs_revert(config, blob, version, get_gitactor(request)) - - -# -# @ns_configs.route("//b//v//revert/") -# class BlobRevertResource(Resource): -# def put(self, config, blob, version): -# "Create a new version for a blob from an old version" -# return current_app.backend.blobs_revert(config, blob, version, get_gitactor()) - - -################# -### DOCUMENTS ### -################# - -@app.get("/configs/{config}/d/", tags=[Tags.congifs], response_model=DocumentListEntry) -async def document_resource(config: str): - """Retrieve the list of existing documents in this configuration""" - res = app.backend.documents_list(config) - return res - - -# -# @ns_configs.route("//d/") -# class DocumentsResource(Resource): -# @ns_configs.marshal_with(m_document_list_entry, skip_none=True) -# def get(self, config): -# "Retrieve the list of existing documents in this configuration" -# res = current_app.backend.documents_list(config) -# return res - - -@app.get("/configs/{config}/d/{document}/", tags=[Tags.congifs], response_model=DocumentMask) -async def document_resource_get(config: str, document: str): - """Get a complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - res = app.backend.documents_get(config, document) - res = {key: res[key] for key in list(models[document].__fields__.keys())} - return res - - -async def _filter(data, keys): - return {key: data[key] for key in keys} - - -@app.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_post(config: str, document: str, basic_entries: List[BasicEntry], request: Request): - """Create a new complete document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) - if isValid is False: - raise HTTPException(500, "schema mismatched: \n" + err) - res = app.backend.documents_create( - config, document, data, get_gitactor(request) - ) - return res - - -@app.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_put(config: str, document: str, basic_entries: List[BasicEntry], request: Request): - """Update an existing document""" - if document not in models: - raise HTTPException(status_code=404, detail="document does not exist") - - data = [_filter(dict(entry), list(models[document].__fields__.keys())) for entry in basic_entries] - for entry in basic_entries: - isValid, err = validateJson(dict(entry), document) - if isValid is False: - raise HTTPException(500, "schema mismatched for entry: " + str(entry) + "\n" + err) - res = app.backend.documents_update( - config, document, data, get_gitactor(request) - ) - return res - - -@app.delete("/configs/{config}/d/{document}/", tags=[Tags.congifs]) -async def document_resource_delete(config: str, document: str, request: Request): - """Delete/empty a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_delete(config, document, get_gitactor(request)) - return res - - -# @ns_configs.route("//d//") -# class DocumentResource(Resource): -# # @ns_configs.marshal_with(m_document_mask, mask="*", skip_none=True) -# # def get(self, config, document): -# # "Get a complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_get(config, document) -# # return marshal(res, models[document], skip_none=True) -# # -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def post(self, config, document): -# # "Create a new complete document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched: \n" + err) -# # res = current_app.backend.documents_create( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # @ns_configs.expect([m_basic_entry], validate=True) -# # def put(self, config, document): -# # "Update an existing document" -# # if document not in models: -# # abort(404, "document does not exist") -# # data = marshal(request.json, models[document], skip_none=True) -# # for entry in request.json: -# # isValid, err = validateJson(entry, document) -# # if isValid is False: -# # abort(500, "schema mismatched for entry: " + str(entry) + "\n" + err) -# # res = current_app.backend.documents_update( -# # config, document, data, get_gitactor() -# # ) -# # return res -# -# # def delete(self, config, document): -# # "Delete/empty a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.documents_delete(config, document, get_gitactor()) -# # return res - - -@app.get("/configs/{config}/d/{document}/v/", tags=[Tags.congifs]) -async def document_list_version_resource_get(config: str, document: str): - """Retrieve the existing versions of a given document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_list_versions(config, document) - res = {key: res[key] for key in list(VersionLog.__fields__.keys())} - return res - - -# -# @ns_configs.route("//d//v/") -# class DocumentListVersionResource(Resource): -# def get(self, config, document): -# "Retrieve the existing versions of a given document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_list_versions(config, document) -# return marshal(res, m_version_log, skip_none=True) - - -@app.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) -async def document_version_resource_get(config: str, document: str, version: str): - """Get a given version of a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.documents_get(config, document, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} - - -# @ns_configs.route("//d//v//") -# class DocumentVersionResource(Resource): -# def get(self, config, document, version): -# "Get a given version of a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.documents_get(config, document, version) -# return marshal(res, models[document], skip_none=True) - -@app.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) -async def document_revert_resource_put(config: str, document: str, version: str, request: Request): - """Create a new version for a document from an old version""" - return app.backend.documents_revert( - config, document, version, get_gitactor(request) - ) - - -# @ns_configs.route("//d//v//revert/") -# class DocumentRevertResource(Resource): -# def put(self, config, document, version): -# "Create a new version for a document from an old version" -# return current_app.backend.documents_revert( -# config, document, version, get_gitactor() -# ) - - -############### -### ENTRIES ### -############### - -@app.get("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_get(config: str, document: str): - """Retrieve the list of entries in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_list(config, document) - return res # XXX: marshal - - -@app.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) -async def entries_resource_post(config: str, document: str, basic_entry: BasicEntry, request: Request): - "Create an entry in a document" - if document not in models: - raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) - if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - res = app.backend.entries_create( - config, document, data, get_gitactor(request) - ) - return res - else: - raise HTTPException(500, "schema mismatched: \n" + err) - - -# @ns_configs.route("//d//e/") -# class EntriesResource(Resource): -# # def get(self, config, document): -# # "Retrieve the list of entries in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_list(config, document) -# # return res # XXX: marshal -# -# @ns_configs.expect(m_basic_entry, validate=True) -# def post(self, config, document): -# "Create an entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# isValid, err = validateJson(request.json, document) -# if isValid: -# data = marshal(request.json, models[document], skip_none=True) -# res = current_app.backend.entries_create( -# config, document, data, get_gitactor() -# ) -# return res -# else: -# abort(500, "schema mismatched: \n" + err) - - -@app.get("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_get(config: str, document: str, entry: str): - """Retrieve an entry from a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry) - return {key: res for key in list(models[document].__fields__.keys())} - - -@app.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_put(config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request): - """Update an entry in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - isValid, err = validateJson(dict(basic_entry), document) - if isValid: - data = {key: dict(basic_entry)[key] for key in list(models[document].__fields__.keys())} - - res = app.backend.entries_update( - config, document, entry, data, get_gitactor(request) - ) - return res - else: - raise HTTPException(500, "schema mismatched: \n" + err) - - -@app.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye(config: str, document: str, entry: str, request: Request): - """Delete an entry from a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_delete( - config, document, entry, get_gitactor(request) - ) - return res - - -# @ns_configs.route("//d//e//") -# class EntryResource(Resource): -# # def get(self, config, document, entry): -# # "Retrieve an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_get(config, document, entry) -# # return marshal(res, models[document], skip_none=True) -# -# # @ns_configs.expect(m_basic_entry, validate=True) -# # def put(self, config, document, entry): -# # "Update an entry in a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # isValid, err = validateJson(request.json, document) -# # if isValid: -# # data = marshal(request.json, models[document], skip_none=True) -# # res = current_app.backend.entries_update( -# # config, document, entry, data, get_gitactor() -# # ) -# # return res -# # else: -# # abort(500, "schema mismatched: \n" + err) -# -# # def delete(self, config, document, entry): -# # "Delete an entry from a document" -# # if document not in models: -# # abort(404, "document does not exist") -# # res = current_app.backend.entries_delete( -# # config, document, entry, get_gitactor() -# # ) -# # return res - - -@app.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) -async def entry_list_version_resource_get(config: str, document: str, entry: str): - """Get the list of existing versions of a given entry in a document""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_list_versions(config, document, entry) - return {key: res[key] for key in list(VersionLog.__fields__.keys())} - - -# -# @ns_configs.route("//d//e//v/") -# class EntryListVersionResource(Resource): -# def get(self, config, document, entry): -# "Get the list of existing versions of a given entry in a document" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_list_versions(config, document, entry) -# return marshal(res, m_version_log, skip_none=True) - - -@app.get("/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs]) -async def entry_version_resource_get(config: str, document: str, entry: str, version: str): - """Get a given version of a document entry""" - if document not in models: - raise HTTPException(404, "document does not exist") - res = app.backend.entries_get(config, document, entry, version) - return {key: res[key] for key in list(models[document].__fields__.keys())} - - -# @ns_configs.route( -# "//d//e//v//" -# ) -# class EntryVersionResource(Resource): -# def get(self, config, document, entry, version): -# "Get a given version of a document entry" -# if document not in models: -# abort(404, "document does not exist") -# res = current_app.backend.entries_get(config, document, entry, version) -# return marshal(res, models[document], skip_none=True) - - -################ -### Database ### -################ - -@app.get("/db/", tags=[Tags.db]) -async def db_resource_get(): - """Get the list of existing namespaces""" - return app.backend.ns_list() - - -# @ns_db.route("/") -# class DbResource(Resource): -# def get(self): -# "Get the list of existing namespaces" -# return current_app.backend.ns_list() - - -@app.get("/db/v/", tags=[Tags.db]) -async def db_query_resource_get(): - """List all existing versions of namespaces""" - return app.backend.ns_list_versions() - - -# -# @ns_db.route("/v/") -# class DbQueryResource(Resource): -# def get(self): -# "List all existing versions of namespaces" -# return current_app.backend.ns_list_versions() - - -@app.get("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_get(nsname: str): - """Get a complete namespace""" - try: - return app.backend.ns_get(nsname, version=None) - except KeyError: - raise HTTPException(404, "namespace [%s] does not exist" % nsname) - - -@app.post("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_post(nsname: str, db: DB, request: Request): - """Create a non-existing namespace from data""" - try: - return app.backend.ns_create(nsname, dict(db), get_gitactor(request)) - except Exception: - raise HTTPException(409, "namespace [%s] already exists" % nsname) - - -@app.put("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_put(nsname: str, db: DB, request: Request): - """Merge data into a namespace""" - return app.backend.ns_update(nsname, dict(db), get_gitactor(request)) - - -@app.delete("/db/{nsname}/", tags=[Tags.db]) -async def ns_resource_put(nsname: str, request: Request): - """Delete an existing namespace""" - try: - return app.backend.ns_delete(nsname, get_gitactor(request)) - except KeyError: - raise HTTPException(409, "namespace [%s] does not exist" % nsname) - - -# @ns_db.route("//") -# class NSResource(Resource): -# # def get(self, nsname): -# # "Get a complete namespace" -# # try: -# # return current_app.backend.ns_get(nsname, version=None) -# # except KeyError: -# # abort(404, "namespace [%s] does not exist" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def post(self, nsname): -# # "Create a non-existing namespace from data" -# # try: -# # return current_app.backend.ns_create(nsname, request.json, get_gitactor()) -# # except Exception: -# # abort(409, "namespace [%s] already exists" % nsname) -# -# # @ns_db.expect(m_db, validate=True) -# # def put(self, nsname): -# # "Merge data into a namespace" -# # return current_app.backend.ns_update(nsname, request.json, get_gitactor()) -# -# # def delete(self, nsname): -# # "Delete an existing namespace" -# # try: -# # return current_app.backend.ns_delete(nsname, get_gitactor()) -# # except KeyError: -# # abort(409, "namespace [%s] does not exist" % nsname) - - -@app.get("/db/{nsname}/v/{version}", tags=[Tags.db]) -async def ns_version_resource_get(nsname: str, version: str): - """Get a given version of a namespace""" - return app.backend.ns_get(nsname, version) - - -# @ns_db.route("//v//") -# class NSVersionResource(Resource): -# def get(self, nsname, version): -# "Get a given version of a namespace" -# return current_app.backend.ns_get(nsname, version) - - -@app.put("/db/{nsname}/v/{version}/revert/", tags=[Tags.db]) -async def ns_version_revert_resource_put(nsname: str, version: str, request: Request): - """Create a new version for a namespace from an old version""" - try: - return app.backend.ns_revert(nsname, version, get_gitactor(request)) - except KeyError: - raise HTTPException(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -# -# @ns_db.route("//v//revert/") -# class NSVersionResource(Resource): -# def put(self, nsname, version): -# "Create a new version for a namespace from an old version" -# try: -# return current_app.backend.ns_revert(nsname, version, get_gitactor()) -# except KeyError: -# abort(404, "namespace [%s] version [%s] not found" % (nsname, version)) - - -@app.post("/db/{nsname}/q/", tags=[Tags.db]) -async def ns_query_resource_post(nsname: str, request: Request): - """Run a JSON query on the namespace and returns the results""" - req_json = await request.json() - return app.backend.ns_query(nsname, req_json) - - -# @ns_db.route("//q/") -# class NSQueryResource(Resource): -# def post(self, nsname): -# "Run a JSON query on the namespace and returns the results" -# return current_app.backend.ns_query(nsname, request.json) -# - - -@app.get("/db/{nsname}/k/", tags=[Tags.db]) -async def keys_resource_get(nsname: str): - """List all keys of a given namespace""" - return app.backend.key_list(nsname) - - -# @ns_db.route("//k/") -# class KeysResource(Resource): -# def get(self, nsname): -# "List all keys of a given namespace" -# return current_app.backend.key_list(nsname) - -@app.get("/db/{nsname}/k/{key}/v/", tags=[Tags.db]) -async def keys_list_versions_resource_get(nsname: str, key: str): - """Get all versions of a given key in namespace""" - return app.backend.key_list_versions(nsname, key) - - -# @ns_db.route("//k//v/") -# class KeysListVersionsResource(Resource): -# def get(self, nsname, key): -# "Get all versions of a given key in namespace" -# return current_app.backend.key_list_versions(nsname, key) -# - -@app.get("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_get(nsname: str, key: str): - """Retrieve a given key's value from a given namespace""" - return app.backend.key_get(nsname, key) - - -@app.put("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_put(nsname: str, key: str, request: Request): - """Create or update the value of a key""" - # check if "reblaze/k/" exists in system/schema-validation - req_json = await request.json() - - if nsname != "system": - keyName = nsname + "/k/" + key - schemas = app.backend.key_get("system", "schema-validation") - schema = None - # find schema if exists and validate the json input - for item in schemas.items(): - if item[0] == keyName: - schema = item[1] - break - if schema: - isValid = validateDbJson(req_json, schema) - if isValid is False: - raise HTTPException(500, "schema mismatched") - return app.backend.key_set(nsname, key, req_json, get_gitactor(request)) - - -@app.delete("/db/{nsname}/k/{key}/", tags=[Tags.db]) -async def key_resource_delete(nsname: str, key: str, request: Request): - """Delete a key""" - return app.backend.key_delete(nsname, key, get_gitactor(request)) - - -# @ns_db.route("//k//") -# class KeyResource(Resource): -# # def get(self, nsname, key): -# # "Retrieve a given key's value from a given namespace" -# # return current_app.backend.key_get(nsname, key) -# -# # def put(self, nsname, key): -# # "Create or update the value of a key" -# # # check if "reblaze/k/" exists in system/schema-validation -# # if nsname != "system": -# # keyName = nsname + "/k/" + key -# # schemas = current_app.backend.key_get("system", "schema-validation") -# # schema = None -# # # find schema if exists and validate the json input -# # for item in schemas.items(): -# # if item[0] == keyName: -# # schema = item[1] -# # break -# # if schema: -# # isValid = validateDbJson(request.json, schema) -# # if isValid is False: -# # abort(500, "schema mismatched") -# # return current_app.backend.key_set(nsname, key, request.json, get_gitactor()) -# -# # def delete(self, nsname, key): -# # "Delete a key" -# # return current_app.backend.key_delete(nsname, key, get_gitactor()) - - -############# -### Tools ### -############# - - -req_fetch_parser = reqparse.RequestParser() -req_fetch_parser.add_argument("url", location="args", help="url to retrieve") - - -@app.get("/tools/fetch", tags=[Tags.tools]) -async def fetch_resource_get(url: str): - """Fetch an URL""" - try: - r = requests.get(url) - except Exception as e: - raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) - return r.content - - -# @ns_tools.route("/fetch") -# class FetchResource(Resource): -# @ns_tools.expect(req_fetch_parser, validate=True) -# def get(self): -# "Fetch an URL" -# args = req_fetch_parser.parse_args() -# try: -# r = requests.get(args.url) -# except Exception as e: -# abort(400, "cannot retrieve [%s]: %s" % (args.url, e)) -# return make_response(r.content) - - -@app.put("/tools/publish/{config}/", tags=[Tags.tools]) -@app.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) -async def publish_resource_put(config: str, request: Request, buckets: List[Bucket], version: str = None): - """Push configuration to s3 buckets""" - conf = app.backend.configs_get(config, version) - ok = True - status = [] - req_json = await request.json() - if type(req_json) is not list: - raise HTTPException(400, "body must be a list") - for bucket in buckets: - logs = [] - try: - cloud.export(conf, dict(bucket)["url"], prnt=lambda x: logs.append(x)) - except Exception as e: - ok = False - s = False - msg = repr(e) - else: - s = True - msg = "ok" - status.append( - {"name": dict(bucket)["name"], "ok": s, "logs": logs, "message": msg} - ) - return {"ok": ok, "status": status} - - -# @ns_tools.route("/publish//") -# @ns_tools.route("/publish//v//") -# class PublishResource(Resource): -# @ns_tools.expect([m_bucket], validate=True) -# def put(self, config, version=None): -# "Push configuration to s3 buckets" -# conf = current_app.backend.configs_get(config, version) -# ok = True -# status = [] -# if type(request.json) is not list: -# abort(400, "body must be a list") -# for bucket in request.json: -# logs = [] -# try: -# cloud.export(conf, bucket["url"], prnt=lambda x: logs.append(x)) -# except Exception as e: -# ok = False -# s = False -# msg = repr(e) -# else: -# s = True -# msg = "ok" -# status.append( -# {"name": bucket["name"], "ok": s, "logs": logs, "message": msg} -# ) -# return make_response({"ok": ok, "status": status}) - - -@app.put("/tools/gitpush/", tags=[Tags.tools]) -async def git_push_resource_put(git_urls: List[GitUrl]): - """Push git configuration to remote git repositories""" - ok = True - status = [] - for giturl in git_urls: - try: - app.backend.gitpush(dict(giturl)["giturl"]) - except Exception as e: - msg = repr(e) - s = False - else: - msg = "ok" - s = True - status.append({"url": dict(giturl)["giturl"], "ok": s, "message": msg}) - return {"ok": ok, "status": status} - - -# @ns_tools.route("/gitpush/") -# class GitPushResource(Resource): -# @ns_tools.expect([m_giturl], validate=True) -# def put(self): -# "Push git configuration to remote git repositories" -# ok = True -# status = [] -# for giturl in request.json: -# try: -# current_app.backend.gitpush(giturl["giturl"]) -# except Exception as e: -# msg = repr(e) -# s = False -# else: -# msg = "ok" -# s = True -# status.append({"url": giturl["giturl"], "ok": s, "message": msg}) -# return make_response({"ok": ok, "status": status}) - - -@app.put("/tools/gitfetch/", tags=[Tags.tools]) -async def git_fetch_resource_put(giturl: GitUrl): - """Fetch git configuration from specified remote repository""" - ok = True - try: - app.backend.gitfetch(dict(giturl)["giturl"]) - except Exception as e: - ok = False - msg = repr(e) - else: - msg = "ok" - return {"ok": ok, "status": msg} - - -# @ns_tools.route("/gitfetch/") -# class GitFetchResource(Resource): -# @ns_tools.expect(m_giturl, validate=True) -# def put(self): -# "Fetch git configuration from specified remote repository" -# ok = True -# try: -# current_app.backend.gitfetch(request.json["giturl"]) -# except Exception as e: -# ok = False -# msg = repr(e) -# else: -# msg = "ok" -# return make_response({"ok": ok, "status": msg}) - - -if __name__ == '__main__': - print("hi") From dd1318fdf12043bd6d705421a5855fa75143d9f2 Mon Sep 17 00:00:00 2001 From: yoavkatzman <100856091+yoavkatzman@users.noreply.github.com> Date: Thu, 8 Dec 2022 11:15:34 +0200 Subject: [PATCH 45/55] Update curiefense-manual-build-image.yml --- .github/workflows/curiefense-manual-build-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/curiefense-manual-build-image.yml b/.github/workflows/curiefense-manual-build-image.yml index d9832fad7..f3f7a5fa5 100644 --- a/.github/workflows/curiefense-manual-build-image.yml +++ b/.github/workflows/curiefense-manual-build-image.yml @@ -29,4 +29,3 @@ jobs: PUSH=1 ./build-docker-images.sh export DOCKER_TAG=dev PUSH=1 ./build-docker-images.sh -Symbols From c3d0d88647b9e7c2193e2696d857b91f1f57205a Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 8 Dec 2022 11:24:13 +0200 Subject: [PATCH 46/55] wildcard fixes Signed-off-by: Yoav Katzman Signed-off-by: Curiefense Bootstrap Script --- .../server/curieconf/confserver/v3/api.py | 153 ++++++++++-------- 1 file changed, 84 insertions(+), 69 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 25e9c5a17..893ff74a6 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,7 +4,7 @@ from typing import Optional, List, Union from fastapi import Request, HTTPException, APIRouter -from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt +from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt, Extra import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type @@ -192,7 +192,7 @@ class VirtualTag(BaseModel): # custom -class Custom(BaseModel): +class Custom(BaseModel, extra=Extra.allow): id: StrictStr name: StrictStr @@ -214,7 +214,7 @@ class Custom(BaseModel): ### Other models -class DocumentMask(BaseModel): +class DocumentMask(BaseModel, extra=Extra.allow): id: StrictStr name: StrictStr description: Optional[StrictStr] @@ -244,15 +244,12 @@ class DocumentMask(BaseModel): type_: Optional[StrictStr] = Field(alias="type") star_: Optional[List[typing.Any]] = Field(alias="*") - class Config: - fields = {"_type": "type"} - -class VersionLog(BaseModel): +class VersionLog(BaseModel, extra=Extra.allow): version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] - star_: Optional[List[typing.Any]] = Field(alias="*") + # star_: Optional[List[typing.Any]] = Field(alias="*") class Meta(BaseModel): @@ -393,7 +390,7 @@ def get_gitactor(request): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -404,7 +401,7 @@ def get_gitactor(request): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -461,10 +458,11 @@ async def configs_post(config: Config, request: Request): return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) -@router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +@router.get("/configs/{config}/", tags=[Tags.congifs]) async def config_get(config: str, request: Request): """Retrieve a complete configuration""" - return request.app.backend.configs_get(config) + res = request.app.backend.configs_get(config) + return {key: res[key] for key in Config.__fields__.keys() if key in res} @router.post("/configs/{config}/", tags=[Tags.congifs]) @@ -496,7 +494,7 @@ async def config_clone_post(config: str, meta: Meta, request: Request): @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) async def config_clone_name_post( - config: str, new_name: str, meta: Meta, request: Request + config: str, new_name: str, meta: Meta, request: Request ): """Clone a configuration. New name is provided URL""" data = await request.json() @@ -518,12 +516,11 @@ def filter_x_fields(res, x_fields): @router.get( "/configs/{config}/v/", tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, ) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" res = request.app.backend.configs_list_versions(config) + if request.headers.get("X-fields", False): res = filter_x_fields(res, request.headers["X-fields"]) return res @@ -565,7 +562,7 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """Create a new blob""" b_entry = await request.json() @@ -576,7 +573,7 @@ async def blob_resource_post( @router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """upaate an existing blob""" b_entry = await request.json() @@ -594,9 +591,7 @@ async def blob_resource_delete(config: str, blob: str, request: Request): @router.get( "/configs/{config}/b/{blob}/v/", - tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, + tags=[Tags.congifs] ) async def blob_list_version_resource_get(config: str, blob: str, request: Request): """Retrieve the list of versions of a given blob""" @@ -612,7 +607,7 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques response_model=BlobEntry, ) async def blob_version_resource_get( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Retrieve the given version of a blob""" @@ -624,7 +619,7 @@ async def blob_version_resource_get( @router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Create a new version for a blob from an old version""" return request.app.backend.blobs_revert( @@ -655,10 +650,12 @@ def filter_document_mask(res, x_fields): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") + return [{field: r[field] for field in fields if field in r} for r in res] else: - fields = switch_alias(DocumentMask.__fields__.keys()) + return res + + - return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" @@ -666,6 +663,7 @@ def filter_document_mask(res, x_fields): if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) + return filter_document_mask(res, headers.get("x-fields", None)) @@ -675,7 +673,7 @@ async def _filter(data, keys): @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Create a new complete document""" if document not in models.keys(): @@ -685,10 +683,12 @@ async def document_resource_post( ) as_dict = await request.json() - data = [ - await _filter(dict(entry), list(models[document].__fields__.keys())) - for entry in as_dict - ] + if document == "custom": + data = as_dict + else: + data = [ + await _filter(dict(entry), list(models[document].__fields__.keys())) + for entry in as_dict] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: @@ -701,16 +701,19 @@ async def document_resource_post( @router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_put( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [ - await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) - for entry in as_dict - ] + if document == "custom": + data = as_dict + else: + data = [ + await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: @@ -735,10 +738,9 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get( "/configs/{config}/d/{document}/v/", tags=[Tags.congifs], - response_model=List[VersionLog], ) async def document_list_version_resource_get( - config: str, document: str, request: Request + config: str, document: str, request: Request ): """Retrieve the existing versions of a given document""" if document not in models: @@ -750,25 +752,28 @@ async def document_list_version_resource_get( @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) async def document_version_resource_get( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [ - { - key: r[key] - for key in switch_alias(list(models[document].__fields__.keys())) - if key in r - } - for r in res - ] + if document == "custom": + return res + else: + return [ + { + key: r[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in r + } + for r in res + ] @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) async def document_revert_resource_put( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Create a new version for a document from an old version""" return request.app.backend.documents_revert( @@ -792,7 +797,7 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_post( - config: str, document: str, basic_entry: BasicEntry, request: Request + config: str, document: str, basic_entry: BasicEntry, request: Request ): "Create an entry in a document" @@ -801,11 +806,16 @@ async def entries_resource_post( raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - keys = switch_alias(list(models[document].__fields__.keys())) - data = {key: data_json[key] for key in keys if key in data_json} - res = request.app.backend.entries_create( - config, document, data, get_gitactor(request) - ) + if document == "custom": + res = request.app.backend.entries_create( + config, document, data_json, get_gitactor(request) + ) + else: + keys = switch_alias(list(models[document].__fields__.keys())) + data = {key: data_json[key] for key in keys if key in data_json} + res = request.app.backend.entries_create( + config, document, data, get_gitactor(request) + ) return res else: raise HTTPException(500, "schema mismatched: \n" + err) @@ -821,13 +831,15 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) + if document == "custom": + return res keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put( - config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request + config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request ): """Update an entry in a document""" data_json = await request.json() @@ -835,11 +847,14 @@ async def entry_resource_put( raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = { - key: data_json[key] - for key in switch_alias(list(models[document].__fields__.keys())) - if key in data_json - } + if document == "custom": + data = data_json + else: + data = { + key: data_json[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in data_json + } res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) @@ -850,8 +865,8 @@ async def entry_resource_put( @router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye( - config: str, document: str, entry: str, request: Request +async def entry_resource_delete( + config: str, document: str, entry: str, request: Request ): """Delete an entry from a document""" if document not in models: @@ -864,15 +879,13 @@ async def entry_resource_deleye( @router.get( "/configs/{config}/d/{document}/e/{entry}/v/", - tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, + tags=[Tags.congifs] ) async def entry_list_version_resource_get( - config: str, - document: str, - entry: str, - request: Request, + config: str, + document: str, + entry: str, + request: Request, ): """Get the list of existing versions of a given entry in a document""" if document not in models: @@ -885,12 +898,14 @@ async def entry_list_version_resource_get( "/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs] ) async def entry_version_resource_get( - config: str, document: str, entry: str, version: str, request: Request + config: str, document: str, entry: str, version: str, request: Request ): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) + if document == "custom": + return res keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @@ -1036,7 +1051,7 @@ async def fetch_resource_get(url: str): @router.put("/tools/publish/{config}/", tags=[Tags.tools]) @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) async def publish_resource_put( - config: str, request: Request, buckets: List[Bucket], version: str = None + config: str, request: Request, buckets: List[Bucket], version: str = None ): """Push configuration to s3 buckets""" conf = request.app.backend.configs_get(config, version) From 9060ec5358ec68a1e253b4c4556fd794cf40bc1a Mon Sep 17 00:00:00 2001 From: Yoav Katzman Date: Thu, 8 Dec 2022 11:24:13 +0200 Subject: [PATCH 47/55] HTTPException handler, schema 400, Signed-off-by: Yoav Katzman Signed-off-by: Curiefense Bootstrap Script --- .../server/curieconf/confserver/__init__.py | 28 +-- .../server/curieconf/confserver/v3/api.py | 164 ++++++++++-------- 2 files changed, 106 insertions(+), 86 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 2baa918ac..453a838b6 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -7,13 +7,13 @@ import logging from curieconf.confserver.v3 import api -from fastapi import FastAPI, Request, status +from fastapi import FastAPI, Request, HTTPException from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse, PlainTextResponse -from fastapi.encoders import jsonable_encoder +from fastapi.responses import PlainTextResponse from prometheus_fastapi_instrumentator import Instrumentator +from werkzeug.exceptions import HTTPException as WerkzeugHTTPException -app = FastAPI(docs_url="/api/v3/") +app = FastAPI(docs_url=os.environ.get("SWAGGER_BASE_PATH", "/api/v3/")) app.include_router(api.router) @@ -25,7 +25,6 @@ async def startup(): ## Import all versions from .v3 import api as api_v3 - logging.basicConfig( handlers=[logging.FileHandler("fastapi.log"), logging.StreamHandler()], level=logging.INFO, @@ -37,13 +36,18 @@ async def startup(): @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): - # exc_str = f'{exc}'.replace('\n', ' ').replace(' ', ' ') - # # or logger.error(f'{exc}') - # logger.error(exc_str) - # content = {'status_code': 10422, 'message': exc_str, 'data': None} - # - # return JSONResponse - return PlainTextResponse(str(exc), status_code=400) + return PlainTextResponse(str(exc), status_code=409) + + +# this is ctaching flasks' "abort" from the gitbackend +@app.exception_handler(WerkzeugHTTPException) +async def werkzeug_exception_handler(request: Request, exc: WerkzeugHTTPException): + return PlainTextResponse(str(exc), status_code=exc.code) + + +@app.exception_handler(HTTPException) +async def http_exception_exception_handler(request: Request, exc: HTTPException): + return PlainTextResponse(str(exc.detail), status_code=exc.status_code) def drop_into_pdb(app, exception): diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 25e9c5a17..cb49784e6 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,7 +4,7 @@ from typing import Optional, List, Union from fastapi import Request, HTTPException, APIRouter -from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt +from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt, Extra import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type @@ -192,7 +192,7 @@ class VirtualTag(BaseModel): # custom -class Custom(BaseModel): +class Custom(BaseModel, extra=Extra.allow): id: StrictStr name: StrictStr @@ -214,7 +214,7 @@ class Custom(BaseModel): ### Other models -class DocumentMask(BaseModel): +class DocumentMask(BaseModel, extra=Extra.allow): id: StrictStr name: StrictStr description: Optional[StrictStr] @@ -244,15 +244,12 @@ class DocumentMask(BaseModel): type_: Optional[StrictStr] = Field(alias="type") star_: Optional[List[typing.Any]] = Field(alias="*") - class Config: - fields = {"_type": "type"} - -class VersionLog(BaseModel): +class VersionLog(BaseModel, extra=Extra.allow): version: Optional[StrictStr] # TODO - dt_format="iso8601" date: Optional[datetime.datetime] - star_: Optional[List[typing.Any]] = Field(alias="*") + # star_: Optional[List[typing.Any]] = Field(alias="*") class Meta(BaseModel): @@ -393,7 +390,7 @@ def get_gitactor(request): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -404,7 +401,7 @@ def get_gitactor(request): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -461,10 +458,11 @@ async def configs_post(config: Config, request: Request): return request.app.backend.configs_create(data=data, actor=get_gitactor(request)) -@router.get("/configs/{config}/", tags=[Tags.congifs], response_model=Config) +@router.get("/configs/{config}/", tags=[Tags.congifs]) async def config_get(config: str, request: Request): """Retrieve a complete configuration""" - return request.app.backend.configs_get(config) + res = request.app.backend.configs_get(config) + return {key: res[key] for key in Config.__fields__.keys() if key in res} @router.post("/configs/{config}/", tags=[Tags.congifs]) @@ -496,7 +494,7 @@ async def config_clone_post(config: str, meta: Meta, request: Request): @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) async def config_clone_name_post( - config: str, new_name: str, meta: Meta, request: Request + config: str, new_name: str, meta: Meta, request: Request ): """Clone a configuration. New name is provided URL""" data = await request.json() @@ -518,12 +516,11 @@ def filter_x_fields(res, x_fields): @router.get( "/configs/{config}/v/", tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, ) async def config_list_version_get(config: str, request: Request): """Get all versions of a given configuration""" res = request.app.backend.configs_list_versions(config) + if request.headers.get("X-fields", False): res = filter_x_fields(res, request.headers["X-fields"]) return res @@ -565,7 +562,7 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """Create a new blob""" b_entry = await request.json() @@ -576,7 +573,7 @@ async def blob_resource_post( @router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """upaate an existing blob""" b_entry = await request.json() @@ -594,9 +591,7 @@ async def blob_resource_delete(config: str, blob: str, request: Request): @router.get( "/configs/{config}/b/{blob}/v/", - tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, + tags=[Tags.congifs] ) async def blob_list_version_resource_get(config: str, blob: str, request: Request): """Retrieve the list of versions of a given blob""" @@ -612,7 +607,7 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques response_model=BlobEntry, ) async def blob_version_resource_get( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Retrieve the given version of a blob""" @@ -624,7 +619,7 @@ async def blob_version_resource_get( @router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Create a new version for a blob from an old version""" return request.app.backend.blobs_revert( @@ -655,10 +650,12 @@ def filter_document_mask(res, x_fields): x_fields = x_fields[1:-1] x_fields = x_fields.replace(" ", "") fields = x_fields.split(",") + return [{field: r[field] for field in fields if field in r} for r in res] else: - fields = switch_alias(DocumentMask.__fields__.keys()) + return res + + - return [{field: r[field] for field in fields if field in r} for r in res] """Get a complete document""" @@ -666,6 +663,7 @@ def filter_document_mask(res, x_fields): if document not in models: raise HTTPException(status_code=404, detail="document does not exist") res = request.app.backend.documents_get(config, document) + return filter_document_mask(res, headers.get("x-fields", None)) @@ -675,7 +673,7 @@ async def _filter(data, keys): @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Create a new complete document""" if document not in models.keys(): @@ -685,14 +683,17 @@ async def document_resource_post( ) as_dict = await request.json() - data = [ - await _filter(dict(entry), list(models[document].__fields__.keys())) - for entry in as_dict - ] + if document == "custom": + data = as_dict + else: + data = [ + await _filter(dict(entry), list(models[document].__fields__.keys())) + for entry in as_dict] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: - raise HTTPException(500, "schema mismatched: \n" + err) + + raise HTTPException(400, "schema mismatched: " + err) res = request.app.backend.documents_create( config, document, data, get_gitactor(request) ) @@ -701,21 +702,24 @@ async def document_resource_post( @router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_put( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Update an existing document""" if document not in models: raise HTTPException(status_code=404, detail="document does not exist") as_dict = await request.json() - data = [ - await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) - for entry in as_dict - ] + if document == "custom": + data = as_dict + else: + data = [ + await _filter(entry, switch_alias(list(models[document].__fields__.keys()))) + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: raise HTTPException( - 500, "schema mismatched for entry: " + str(entry) + "\n" + err + 400, "schema mismatched for entry: " + str(entry) + "\n" + err ) res = request.app.backend.documents_update( config, document, data, get_gitactor(request) @@ -735,10 +739,9 @@ async def document_resource_delete(config: str, document: str, request: Request) @router.get( "/configs/{config}/d/{document}/v/", tags=[Tags.congifs], - response_model=List[VersionLog], ) async def document_list_version_resource_get( - config: str, document: str, request: Request + config: str, document: str, request: Request ): """Retrieve the existing versions of a given document""" if document not in models: @@ -750,25 +753,28 @@ async def document_list_version_resource_get( @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) async def document_version_resource_get( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Get a given version of a document""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.documents_get(config, document, version) - return [ - { - key: r[key] - for key in switch_alias(list(models[document].__fields__.keys())) - if key in r - } - for r in res - ] + if document == "custom": + return res + else: + return [ + { + key: r[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in r + } + for r in res + ] @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) async def document_revert_resource_put( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Create a new version for a document from an old version""" return request.app.backend.documents_revert( @@ -792,23 +798,28 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_post( - config: str, document: str, basic_entry: BasicEntry, request: Request + config: str, document: str, basic_entry: BasicEntry, request: Request ): - "Create an entry in a document" + """Create an entry in a document""" data_json = await request.json() if document not in models: raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - keys = switch_alias(list(models[document].__fields__.keys())) - data = {key: data_json[key] for key in keys if key in data_json} - res = request.app.backend.entries_create( - config, document, data, get_gitactor(request) - ) + if document == "custom": + res = request.app.backend.entries_create( + config, document, data_json, get_gitactor(request) + ) + else: + keys = switch_alias(list(models[document].__fields__.keys())) + data = {key: data_json[key] for key in keys if key in data_json} + res = request.app.backend.entries_create( + config, document, data, get_gitactor(request) + ) return res else: - raise HTTPException(500, "schema mismatched: \n" + err) + raise HTTPException(400, "schema mismatched: \n" + err) def switch_alias(keys): @@ -821,13 +832,15 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry) + if document == "custom": + return res keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put( - config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request + config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request ): """Update an entry in a document""" data_json = await request.json() @@ -835,23 +848,26 @@ async def entry_resource_put( raise HTTPException(404, "document does not exist") isValid, err = validateJson(data_json, document) if isValid: - data = { - key: data_json[key] - for key in switch_alias(list(models[document].__fields__.keys())) - if key in data_json - } + if document == "custom": + data = data_json + else: + data = { + key: data_json[key] + for key in switch_alias(list(models[document].__fields__.keys())) + if key in data_json + } res = request.app.backend.entries_update( config, document, entry, data, get_gitactor(request) ) return res else: - raise HTTPException(500, "schema mismatched: \n" + err) + raise HTTPException(400, "schema mismatched: \n" + err) @router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) -async def entry_resource_deleye( - config: str, document: str, entry: str, request: Request +async def entry_resource_delete( + config: str, document: str, entry: str, request: Request ): """Delete an entry from a document""" if document not in models: @@ -864,15 +880,13 @@ async def entry_resource_deleye( @router.get( "/configs/{config}/d/{document}/e/{entry}/v/", - tags=[Tags.congifs], - response_model=List[VersionLog], - response_model_exclude_unset=True, + tags=[Tags.congifs] ) async def entry_list_version_resource_get( - config: str, - document: str, - entry: str, - request: Request, + config: str, + document: str, + entry: str, + request: Request, ): """Get the list of existing versions of a given entry in a document""" if document not in models: @@ -885,12 +899,14 @@ async def entry_list_version_resource_get( "/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs] ) async def entry_version_resource_get( - config: str, document: str, entry: str, version: str, request: Request + config: str, document: str, entry: str, version: str, request: Request ): """Get a given version of a document entry""" if document not in models: raise HTTPException(404, "document does not exist") res = request.app.backend.entries_get(config, document, entry, version) + if document == "custom": + return res keys = switch_alias(list(models[document].__fields__.keys())) return {key: res[key] for key in keys if key in res} @@ -1036,7 +1052,7 @@ async def fetch_resource_get(url: str): @router.put("/tools/publish/{config}/", tags=[Tags.tools]) @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) async def publish_resource_put( - config: str, request: Request, buckets: List[Bucket], version: str = None + config: str, request: Request, buckets: List[Bucket], version: str = None ): """Push configuration to s3 buckets""" conf = request.app.backend.configs_get(config, version) From 5c9f2c7a98d1d5e211d1228beffcdb4fb97bace7 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Wed, 14 Dec 2022 15:24:39 +0200 Subject: [PATCH 48/55] black Signed-off-by: Yoav Katzman Signed-off-by: Curiefense Bootstrap Script --- .../server/curieconf/confserver/v3/api.py | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index cb49784e6..438ceaecd 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,7 +4,14 @@ from typing import Optional, List, Union from fastapi import Request, HTTPException, APIRouter -from pydantic import BaseModel, Field, validator, StrictStr, StrictBool, StrictInt, Extra +from pydantic import ( + BaseModel, + Field, + StrictStr, + StrictBool, + StrictInt, + Extra, +) import jsonschema # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type @@ -390,7 +397,7 @@ def get_gitactor(request): with open(securitypolicies_file_path) as json_file: securitypolicies_schema = json.load(json_file) content_filter_profile_file_path = ( - base_path / "./json/content-filter-profile.schema" + base_path / "./json/content-filter-profile.schema" ).resolve() with open(content_filter_profile_file_path) as json_file: content_filter_profile_schema = json.load(json_file) @@ -401,7 +408,7 @@ def get_gitactor(request): with open(flowcontrol_file_path) as json_file: flowcontrol_schema = json.load(json_file) content_filter_rule_file_path = ( - base_path / "./json/content-filter-rule.schema" + base_path / "./json/content-filter-rule.schema" ).resolve() with open(content_filter_rule_file_path) as json_file: content_filter_rule_schema = json.load(json_file) @@ -494,7 +501,7 @@ async def config_clone_post(config: str, meta: Meta, request: Request): @router.post("/configs/{config}/clone/{new_name}/", tags=[Tags.congifs]) async def config_clone_name_post( - config: str, new_name: str, meta: Meta, request: Request + config: str, new_name: str, meta: Meta, request: Request ): """Clone a configuration. New name is provided URL""" data = await request.json() @@ -562,7 +569,7 @@ async def blob_resource_get(config: str, blob: str, request: Request): @router.post("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_post( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """Create a new blob""" b_entry = await request.json() @@ -573,7 +580,7 @@ async def blob_resource_post( @router.put("/configs/{config}/b/{blob}/", tags=[Tags.congifs]) async def blob_resource_put( - config: str, blob: str, blob_entry: BlobEntry, request: Request + config: str, blob: str, blob_entry: BlobEntry, request: Request ): """upaate an existing blob""" b_entry = await request.json() @@ -589,10 +596,7 @@ async def blob_resource_delete(config: str, blob: str, request: Request): return request.app.backend.blobs_delete(config, blob, get_gitactor(request)) -@router.get( - "/configs/{config}/b/{blob}/v/", - tags=[Tags.congifs] -) +@router.get("/configs/{config}/b/{blob}/v/", tags=[Tags.congifs]) async def blob_list_version_resource_get(config: str, blob: str, request: Request): """Retrieve the list of versions of a given blob""" res = request.app.backend.blobs_list_versions(config, blob) @@ -607,7 +611,7 @@ async def blob_list_version_resource_get(config: str, blob: str, request: Reques response_model=BlobEntry, ) async def blob_version_resource_get( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Retrieve the given version of a blob""" @@ -619,7 +623,7 @@ async def blob_version_resource_get( @router.put("/configs/{config}/b/{blob}/v/{version}/revert/", tags=[Tags.congifs]) async def blob_revert_resource_put( - config: str, blob: str, version: str, request: Request + config: str, blob: str, version: str, request: Request ): """Create a new version for a blob from an old version""" return request.app.backend.blobs_revert( @@ -654,9 +658,6 @@ def filter_document_mask(res, x_fields): else: return res - - - """Get a complete document""" headers = request.headers @@ -673,7 +674,7 @@ async def _filter(data, keys): @router.post("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_post( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Create a new complete document""" if document not in models.keys(): @@ -688,7 +689,8 @@ async def document_resource_post( else: data = [ await _filter(dict(entry), list(models[document].__fields__.keys())) - for entry in as_dict] + for entry in as_dict + ] for entry in data: isValid, err = validateJson(entry, document) if isValid is False: @@ -702,7 +704,7 @@ async def document_resource_post( @router.put("/configs/{config}/d/{document}/", tags=[Tags.congifs]) async def document_resource_put( - config: str, document: str, basic_entries: List[BasicEntry], request: Request + config: str, document: str, basic_entries: List[BasicEntry], request: Request ): """Update an existing document""" if document not in models: @@ -741,7 +743,7 @@ async def document_resource_delete(config: str, document: str, request: Request) tags=[Tags.congifs], ) async def document_list_version_resource_get( - config: str, document: str, request: Request + config: str, document: str, request: Request ): """Retrieve the existing versions of a given document""" if document not in models: @@ -753,7 +755,7 @@ async def document_list_version_resource_get( @router.get("/configs/{config}/d/{document}/v/{version}/", tags=[Tags.congifs]) async def document_version_resource_get( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Get a given version of a document""" if document not in models: @@ -774,7 +776,7 @@ async def document_version_resource_get( @router.put("/configs/{config}/d/{document}/v/{version}/revert/", tags=[Tags.congifs]) async def document_revert_resource_put( - config: str, document: str, version: str, request: Request + config: str, document: str, version: str, request: Request ): """Create a new version for a document from an old version""" return request.app.backend.documents_revert( @@ -798,7 +800,7 @@ async def entries_resource_get(config: str, document: str, request: Request): @router.post("/configs/{config}/d/{document}/e/", tags=[Tags.congifs]) async def entries_resource_post( - config: str, document: str, basic_entry: BasicEntry, request: Request + config: str, document: str, basic_entry: BasicEntry, request: Request ): """Create an entry in a document""" @@ -840,7 +842,7 @@ async def entry_resource_get(config: str, document: str, entry: str, request: Re @router.put("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_put( - config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request + config: str, document: str, entry: str, basic_entry: BasicEntry, request: Request ): """Update an entry in a document""" data_json = await request.json() @@ -867,7 +869,7 @@ async def entry_resource_put( @router.delete("/configs/{config}/d/{document}/e/{entry}/", tags=[Tags.congifs]) async def entry_resource_delete( - config: str, document: str, entry: str, request: Request + config: str, document: str, entry: str, request: Request ): """Delete an entry from a document""" if document not in models: @@ -878,15 +880,12 @@ async def entry_resource_delete( return res -@router.get( - "/configs/{config}/d/{document}/e/{entry}/v/", - tags=[Tags.congifs] -) +@router.get("/configs/{config}/d/{document}/e/{entry}/v/", tags=[Tags.congifs]) async def entry_list_version_resource_get( - config: str, - document: str, - entry: str, - request: Request, + config: str, + document: str, + entry: str, + request: Request, ): """Get the list of existing versions of a given entry in a document""" if document not in models: @@ -899,7 +898,7 @@ async def entry_list_version_resource_get( "/configs/{config}/d/{document}/e/{entry}/v/{version}/", tags=[Tags.congifs] ) async def entry_version_resource_get( - config: str, document: str, entry: str, version: str, request: Request + config: str, document: str, entry: str, version: str, request: Request ): """Get a given version of a document entry""" if document not in models: @@ -1052,7 +1051,7 @@ async def fetch_resource_get(url: str): @router.put("/tools/publish/{config}/", tags=[Tags.tools]) @router.put("/tools/publish/{config}/v/{version}/", tags=[Tags.tools]) async def publish_resource_put( - config: str, request: Request, buckets: List[Bucket], version: str = None + config: str, request: Request, buckets: List[Bucket], version: str = None ): """Push configuration to s3 buckets""" conf = request.app.backend.configs_get(config, version) From 56bb22b67bca33935b70628e3e91b70cfeb4f9ea Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Tue, 20 Dec 2022 16:28:31 +0200 Subject: [PATCH 49/55] git-fix signed Signed-off-by: yoavkatzman --- curiefense/images/confserver/bootstrap/bootstrap_config.sh | 2 ++ curiefense/images/confserver/init/entrypoint.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/curiefense/images/confserver/bootstrap/bootstrap_config.sh b/curiefense/images/confserver/bootstrap/bootstrap_config.sh index 6d0a7a4a5..571b2a8b0 100755 --- a/curiefense/images/confserver/bootstrap/bootstrap_config.sh +++ b/curiefense/images/confserver/bootstrap/bootstrap_config.sh @@ -19,6 +19,8 @@ if [ -n "$IF_NO_CONFIG_PULL_FROM" ]; then fi if [ -n "$IF_NO_CONFIG_INIT_FROM" ]; then + git config --global --add safe.directory $TARGETDIR + git config --global --add safe.directory /cf-persistent-config/bootstrap-repo mkdir -p "$TARGETDIR" cd "$TARGETDIR" git init --bare diff --git a/curiefense/images/confserver/init/entrypoint.sh b/curiefense/images/confserver/init/entrypoint.sh index e703fee3e..8e5b26499 100755 --- a/curiefense/images/confserver/init/entrypoint.sh +++ b/curiefense/images/confserver/init/entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/bash +# make sure git trusts the persistennt directories. +git config --global --add safe.directory /cf-persistent-config/confdb +git config --global --add safe.directory /cf-persistent-config/bootstrap-repo + if [ "$INIT_GIT_ON_STARTUP" = "yes" ]; then # used when running with docker-compose, which does not have initContainers or similar features /bootstrap/bootstrap_config.sh From fede3785a85b84d258c228e967f3b1cf6ea4c07f Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Mon, 27 Feb 2023 10:49:49 +0200 Subject: [PATCH 50/55] black Signed-off-by: yoavkatzman --- curiefense/curieconf/server/curieconf/confserver/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index 453a838b6..f25a9377a 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -39,7 +39,8 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return PlainTextResponse(str(exc), status_code=409) -# this is ctaching flasks' "abort" from the gitbackend + +# this is catching flasks' "abort" from the gitbackend @app.exception_handler(WerkzeugHTTPException) async def werkzeug_exception_handler(request: Request, exc: WerkzeugHTTPException): return PlainTextResponse(str(exc), status_code=exc.code) @@ -49,7 +50,6 @@ async def werkzeug_exception_handler(request: Request, exc: WerkzeugHTTPExceptio async def http_exception_exception_handler(request: Request, exc: HTTPException): return PlainTextResponse(str(exc.detail), status_code=exc.status_code) - def drop_into_pdb(app, exception): import sys import pdb From 46665270e023d244126e9a5f929594f0814fc764 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Mon, 27 Feb 2023 11:14:39 +0200 Subject: [PATCH 51/55] black Signed-off-by: yoavkatzman --- curiefense/curieconf/server/curieconf/confserver/__init__.py | 2 +- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/__init__.py b/curiefense/curieconf/server/curieconf/confserver/__init__.py index f25a9377a..8af1bfe17 100644 --- a/curiefense/curieconf/server/curieconf/confserver/__init__.py +++ b/curiefense/curieconf/server/curieconf/confserver/__init__.py @@ -39,7 +39,6 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return PlainTextResponse(str(exc), status_code=409) - # this is catching flasks' "abort" from the gitbackend @app.exception_handler(WerkzeugHTTPException) async def werkzeug_exception_handler(request: Request, exc: WerkzeugHTTPException): @@ -50,6 +49,7 @@ async def werkzeug_exception_handler(request: Request, exc: WerkzeugHTTPExceptio async def http_exception_exception_handler(request: Request, exc: HTTPException): return PlainTextResponse(str(exc.detail), status_code=exc.status_code) + def drop_into_pdb(app, exception): import sys import pdb diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 438ceaecd..6bd11db34 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -694,7 +694,6 @@ async def document_resource_post( for entry in data: isValid, err = validateJson(entry, document) if isValid is False: - raise HTTPException(400, "schema mismatched: " + err) res = request.app.backend.documents_create( config, document, data, get_gitactor(request) From 50f2e9b33128b7859fdbeee355791402c829d0e0 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Tue, 28 Feb 2023 18:06:30 +0200 Subject: [PATCH 52/55] env variables Signed-off-by: Curiefense Bootstrap Script --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 7 +++++++ curiefense/curieconf/server/setup.py | 1 + 2 files changed, 8 insertions(+) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 6bd11db34..800295e65 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -13,6 +13,7 @@ Extra, ) import jsonschema +import bleach # monkey patch to force RestPlus to use Draft3 validator to benefit from "any" json type jsonschema.Draft4Validator = jsonschema.Draft3Validator @@ -1041,7 +1042,13 @@ async def key_resource_delete(nsname: str, key: str, request: Request): async def fetch_resource_get(url: str): """Fetch an URL""" try: + if not url.startswith("https://"): + raise HTTPException(400, "forbidden url") r = requests.get(url) + r_string = r.content.decode() + if not bleach.clean(r_string) == r_string: + raise HTTPException(400, "forbidden url") + except Exception as e: raise HTTPException(400, "cannot retrieve [%s]: %s" % (url, e)) return r.content diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index d5a5fee86..9ad2718a1 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -37,6 +37,7 @@ "prometheus-fastapi-instrumentator==5.9.1", "pydantic==1.10.2", "uvicorn==0.19.0", + "bleach==6.0.0" ], classifiers=[ "Programming Language :: Python :: 3", From 5a4136781fb01ad95ea236fa73b7f677d9144476 Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Tue, 28 Feb 2023 18:12:32 +0200 Subject: [PATCH 53/55] black Signed-off-by: Curiefense Bootstrap Script --- curiefense/curieconf/server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curiefense/curieconf/server/setup.py b/curiefense/curieconf/server/setup.py index 9ad2718a1..004a419f0 100755 --- a/curiefense/curieconf/server/setup.py +++ b/curiefense/curieconf/server/setup.py @@ -37,7 +37,7 @@ "prometheus-fastapi-instrumentator==5.9.1", "pydantic==1.10.2", "uvicorn==0.19.0", - "bleach==6.0.0" + "bleach==6.0.0", ], classifiers=[ "Programming Language :: Python :: 3", From 1f07c3800a222c98fc6e7a97c8c6ed58a1673544 Mon Sep 17 00:00:00 2001 From: yoavkatzman Date: Thu, 2 Mar 2023 12:14:11 +0200 Subject: [PATCH 54/55] HttpURL Signed-off-by: Curiefense Bootstrap Script --- curiefense/curieconf/server/curieconf/confserver/v3/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 800295e65..51f5e3446 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -11,6 +11,7 @@ StrictBool, StrictInt, Extra, + HttpUrl ) import jsonschema import bleach @@ -1039,7 +1040,7 @@ async def key_resource_delete(nsname: str, key: str, request: Request): @router.get("/tools/fetch", tags=[Tags.tools]) -async def fetch_resource_get(url: str): +async def fetch_resource_get(url: HttpUrl): """Fetch an URL""" try: if not url.startswith("https://"): From d473c4a237023cd45fe16e18d24a87c4b8297b1e Mon Sep 17 00:00:00 2001 From: Curiefense Bootstrap Script Date: Thu, 2 Mar 2023 12:15:59 +0200 Subject: [PATCH 55/55] black Signed-off-by: Curiefense Bootstrap Script --- .../curieconf/server/curieconf/confserver/v3/api.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/curiefense/curieconf/server/curieconf/confserver/v3/api.py b/curiefense/curieconf/server/curieconf/confserver/v3/api.py index 51f5e3446..b0a6d2e96 100644 --- a/curiefense/curieconf/server/curieconf/confserver/v3/api.py +++ b/curiefense/curieconf/server/curieconf/confserver/v3/api.py @@ -4,15 +4,7 @@ from typing import Optional, List, Union from fastapi import Request, HTTPException, APIRouter -from pydantic import ( - BaseModel, - Field, - StrictStr, - StrictBool, - StrictInt, - Extra, - HttpUrl -) +from pydantic import BaseModel, Field, StrictStr, StrictBool, StrictInt, Extra, HttpUrl import jsonschema import bleach