From fc0ab48308f15298cafebaefff2abb26c6d2f976 Mon Sep 17 00:00:00 2001 From: Santhosh Shanmugam Date: Fri, 22 Jul 2022 14:32:39 +0100 Subject: [PATCH 01/10] Added cloud_logger feature --- firetail/auditor.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 1a0fae7..8c7f0dd 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -10,9 +10,22 @@ from .logger import get_stdout_logger +DEFAULT_OPTIONS = dict(origins='*', + methods=['GET', 'HEAD', 'POST', 'OPTIONS', 'PUT', 'PATCH', 'DELETE'], + allow_headers='*', + expose_headers=None, + supports_credentials=False, + max_age=None, + send_wildcard=False, + automatic_options=True, + vary_header=True, + resources=r'/*', + intercept_exceptions=True, + always_send=True) -class auditor: +class cloud_logger(object): def __init__(self, + app, url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds', debug=False, @@ -72,6 +85,13 @@ def __init__(self, } } } + if app: + self.init_app(app, token) + + def init_app(self, app, token): + create_after_request = make_after_request_function(self,token) + app.after_request(create_after_request) + def set_token(self, token_secret): self.token = token_secret @@ -150,5 +170,9 @@ def create(self, response, token, scrub_headers=None, debug=False): pass return payload +def make_after_request_function(cl,token): + def logs_after_request(resp): + cl.create(resp, token) + return resp + return logs_after_request -request_auditor = auditor() From fb87ed3cf559e6e000cdb1dada9e0f00a07d195e Mon Sep 17 00:00:00 2001 From: Santhosh Shanmugam Date: Fri, 22 Jul 2022 16:55:42 +0100 Subject: [PATCH 02/10] Changed references to pointsec to firetail, Added /swagger2 to examples --- .gitignore | 3 +- README.rst | 30 +++++++++---------- docs/conf.py | 8 ++--- docs/security.rst | 10 +++---- examples/openapi3/jwt/app.py | 2 +- examples/openapi3/jwt/requirements.txt | 2 +- examples/openapi3/sqlalchemy/requirements.txt | 2 +- examples/swagger2/oauth2/README.rst | 25 ++++++++++++++++ examples/swagger2/oauth2/app.py | 16 ++++++++++ examples/swagger2/oauth2/app.yaml | 30 +++++++++++++++++++ examples/swagger2/oauth2/mock_tokeninfo.py | 30 +++++++++++++++++++ examples/swagger2/oauth2/mock_tokeninfo.yaml | 23 ++++++++++++++ .../oauth2_local_tokeninfo/README.rst | 24 +++++++++++++++ .../swagger2/oauth2_local_tokeninfo/app.py | 26 ++++++++++++++++ .../swagger2/oauth2_local_tokeninfo/app.yaml | 28 +++++++++++++++++ firetail/auditor.py | 14 ++++----- firetail/exceptions.py | 2 +- firetail/flusher.py | 2 +- firetail/handlers.py | 24 +++++++-------- firetail/sender.py | 26 ++++++++-------- setup.py | 4 +-- 21 files changed, 267 insertions(+), 64 deletions(-) create mode 100644 examples/swagger2/oauth2/README.rst create mode 100755 examples/swagger2/oauth2/app.py create mode 100644 examples/swagger2/oauth2/app.yaml create mode 100755 examples/swagger2/oauth2/mock_tokeninfo.py create mode 100644 examples/swagger2/oauth2/mock_tokeninfo.yaml create mode 100644 examples/swagger2/oauth2_local_tokeninfo/README.rst create mode 100755 examples/swagger2/oauth2_local_tokeninfo/app.py create mode 100644 examples/swagger2/oauth2_local_tokeninfo/app.yaml diff --git a/.gitignore b/.gitignore index 88d9561..2a503b1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ htmlcov/ venv/ .venv/ src/ -*.un~ \ No newline at end of file +*.un~ +*.DS_Store \ No newline at end of file diff --git a/README.rst b/README.rst index 4606d82..a4e1519 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ Firetail =========== -.. _Firetail's Documentation Page: https://pointsecio.readthedocs.org/en/latest/ +.. _Firetail's Documentation Page: https://firetail.readthedocs.org/en/latest/ .. _Connexion: https://github.com/spec-first/connexion .. _Flask: https://flask.pocoo.org/ .. _issues waffle board: https://waffle.io/zalando/connexion @@ -22,35 +22,35 @@ Firetail .. _werkzeug: https://werkzeug.pocoo.org/ .. _Connexion's Documentation Page: https://connexion.readthedocs.org/en/latest/ .. _Crafting effective Microservices in Python: https://jobs.zalando.com/tech/blog/crafting-effective-microservices-in-python/ -.. _issues where we are looking for contributions: https://github.com/pointSec-io/pointsecio/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 +.. _issues where we are looking for contributions: https://github.com/FireTail-io/firetail-py-lib/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _HTTP Methods work in Flask: https://flask.pocoo.org/docs/1.0/quickstart/#http-methods .. .. image:: https://badges.gitter.im/zalando/connexion.svg .. :alt: Join the chat at https://gitter.im/zalando/connexion .. :target: https://gitter.im/zalando/connexion?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. image:: https://github.com/PointSec-io/pointsecio/actions/workflows/pipeline.yml/badge.svg +.. image:: https://github.com/FireTail-io/firetail-py-lib/actions/workflows/pipeline.yml/badge.svg :alt: Build status - :target: https://github.com/PointSec-io/pointsecio/actions/workflows/pipeline.yml + :target: https://github.com/FireTail-io/firetail-py-lib/actions/workflows/pipeline.yml .. .. image:: https://coveralls.io/repos/github/zalando/connexion/badge.svg?branch=main .. :target: https://coveralls.io/github/zalando/connexion?branch=main .. :alt: Coveralls status -.. image:: https://img.shields.io/pypi/v/pointsecio.svg - :target: https://pypi.python.org/pypi/pointsecio +.. image:: https://img.shields.io/pypi/v/firetail.svg + :target: https://pypi.python.org/pypi/firetail :alt: Latest Version -.. image:: https://img.shields.io/pypi/status/pointsecio.svg - :target: https://pypi.python.org/pypi/pointsecio +.. image:: https://img.shields.io/pypi/status/firetail.svg + :target: https://pypi.python.org/pypi/firetail :alt: Development Status -.. image:: https://img.shields.io/pypi/pyversions/pointsecio.svg - :target: https://pypi.python.org/pypi/pointsecio +.. image:: https://img.shields.io/pypi/pyversions/firetail.svg + :target: https://pypi.python.org/pypi/firetail :alt: Python Versions -.. image:: https://img.shields.io/pypi/l/pointsecio.svg - :target: https://github.com/PointSec-io/pointsecio/blob/main/LICENSE.txt +.. image:: https://img.shields.io/pypi/l/firetail.svg + :target: https://github.com/FireTail-io/firetail-py-lib/blob/main/LICENSE.txt :alt: License Firetail (fork of Connexion_) is a framework that automagically handles HTTP requests based on `OpenAPI Specification`_ @@ -582,7 +582,7 @@ Changes A full changelog is maintained on the `GitHub releases page`_. -.. _GitHub releases page: https://github.com/PointSec-io/pointsecio/releases +.. _GitHub releases page: https://github.com/FireTail-io/firetail-py-lib/releases Contributing to Firetail/TODOs ================================ @@ -592,7 +592,7 @@ usual/standard GitHub practices. Unless you explicitly state otherwise in advance, any non trivial contribution intentionally submitted for inclusion in this project by you -to the steward of this repository (point security inc) shall be under the +to the steward of this repository (Point Security Inc DBA FireTail (TM)) shall be under the terms and conditions of Lesser General Public License 2.0 written below, without any additional copyright information, terms or conditions. @@ -604,7 +604,7 @@ TODOs License =================== -Copyright 2022 point security inc +Copyright 2022 Point Security Inc DBA FireTail (TM) Licensed under the Lesser General Public License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.txt. diff --git a/docs/conf.py b/docs/conf.py index f2af2ff..3353b31 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,7 +57,7 @@ # General information about the project. project = 'Firetail' -copyright = '2022, point security inc' +copyright = '2022, Point Security Inc DBA FireTail (TM)' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -217,7 +217,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'Firetail.tex', 'Firetail Documentation', - 'point security inc', 'manual'), + 'Point Security Inc DBA FireTail (TM)', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -247,7 +247,7 @@ # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'firetail', 'Firetail Documentation', - ['point security inc'], 1) + ['Point Security Inc DBA FireTail (TM)'], 1) ] # If true, show URL addresses after external links. @@ -261,7 +261,7 @@ # dir menu entry, description, category) texinfo_documents = [ ('index', 'Firetail', 'Firetail Documentation', - 'point security inc', 'Firetail', 'One line description of project.', + 'Point Security Inc DBA FireTail (TM)', 'Firetail', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/security.rst b/docs/security.rst index 120ed62..7017568 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -48,8 +48,8 @@ You can find a `minimal OAuth example application`_ showing the use of ``x-tokenInfoUrl``, and `another OAuth example`_ showing the use of ``x-tokenInfoFunc`` in Firetail's "examples" folder. -.. _minimal OAuth example application: https://github.com/pointSec-io/pointsecio/tree/main/examples/swagger2/oauth2 -.. _another OAuth example: https://github.com/pointSec-io/pointsecio/tree/main/examples/swagger2/oauth2_local_tokeninfo +.. _minimal OAuth example application: https://github.com/FireTail-io/firetail-py-lib/tree/main/examples/swagger2/oauth2 +.. _another OAuth example: https://github.com/FireTail-io/firetail-py-lib/tree/main/examples/swagger2/oauth2_local_tokeninfo Basic Authentication -------------------- @@ -62,7 +62,7 @@ parameters: username, password and required_scopes. You can find a `minimal Basic Auth example application`_ in Firetail's "examples" folder. .. _oauth scope: https://oauth.net/2/scope/ -.. _minimal Basic Auth example application: https://github.com/pointSec-io/pointsecio/tree/main/examples/openapi3/basicauth +.. _minimal Basic Auth example application: https://github.com/FireTail-io/firetail-py-lib/tree/main/examples/openapi3/basicauth ApiKey Authentication --------------------- @@ -134,7 +134,7 @@ parameters to the underlying `werkzeug`_ server. .. _rfc6750: https://tools.ietf.org/html/rfc6750 .. _rfc6749: https://tools.ietf.org/html/rfc6749 .. _rfc7662: https://tools.ietf.org/html/rfc7662 -.. _minimal API Key example application: https://github.com/pointSec-io/pointsecio/blob/main/examples/openapi3/apikey -.. _minimal JWT example application: https://github.com/pointSec-io/pointsecio/tree/main/examples/openapi3/jwt +.. _minimal API Key example application: https://github.com/FireTail-io/firetail-py-lib/blob/main/examples/openapi3/apikey +.. _minimal JWT example application: https://github.com/FireTail-io/firetail-py-lib/tree/main/examples/openapi3/jwt .. _enabling authentication passthrough in modwsgi: https://modwsgi.readthedocs.io/en/develop/configuration-directives/WSGIPassAuthorization.html .. _modwsgi documentation: https://modwsgi.readthedocs.io/en/develop/index.html diff --git a/examples/openapi3/jwt/app.py b/examples/openapi3/jwt/app.py index fdc4481..55efacb 100644 --- a/examples/openapi3/jwt/app.py +++ b/examples/openapi3/jwt/app.py @@ -9,7 +9,7 @@ from jose import JWTError, jwt from werkzeug.exceptions import Unauthorized -JWT_ISSUER = 'io.pointsec.firetail' +JWT_ISSUER = 'io.firetail.firetail' JWT_SECRET = 'change_this' JWT_LIFETIME_SECONDS = 600 JWT_ALGORITHM = 'HS256' diff --git a/examples/openapi3/jwt/requirements.txt b/examples/openapi3/jwt/requirements.txt index 77e097c..37257c4 100644 --- a/examples/openapi3/jwt/requirements.txt +++ b/examples/openapi3/jwt/requirements.txt @@ -1,6 +1,6 @@ # Install swagger-ui before firetail. firetail[swagger-ui] -firetail>=2.2.0 +firetail>=1.0.3 python-jose[cryptography] Flask>=0.10.1 diff --git a/examples/openapi3/sqlalchemy/requirements.txt b/examples/openapi3/sqlalchemy/requirements.txt index 051156f..2a51da4 100644 --- a/examples/openapi3/sqlalchemy/requirements.txt +++ b/examples/openapi3/sqlalchemy/requirements.txt @@ -1,3 +1,3 @@ -firetail>=1.0.97 +firetail>=1.0.3 Flask>=0.10.1 SQLAlchemy>=1.0.13 \ No newline at end of file diff --git a/examples/swagger2/oauth2/README.rst b/examples/swagger2/oauth2/README.rst new file mode 100644 index 0000000..7ab5851 --- /dev/null +++ b/examples/swagger2/oauth2/README.rst @@ -0,0 +1,25 @@ +============== +OAuth2 Example +============== + +This example demonstrates how to implement a resource server with Connexion. +The app will lookup OAuth2 Bearer tokens with the given token info function. + +Running: + +.. code-block:: bash + + $ sudo pip3 install --upgrade connexion # install Connexion from PyPI + $ ./mock_tokeninfo.py & # start mock in background + $ ./app.py + +Now open your browser and go to http://localhost:8080/ui/ to see the Swagger UI. + +You can use the hardcoded tokens to request the endpoint: + +.. code-block:: bash + + $ curl http://localhost:8080/secret # missing authentication + $ curl -H 'Authorization: Bearer 123' http://localhost:8080/secret + $ curl -H 'Authorization: Bearer 456' http://localhost:8080/secret + diff --git a/examples/swagger2/oauth2/app.py b/examples/swagger2/oauth2/app.py new file mode 100755 index 0000000..7450616 --- /dev/null +++ b/examples/swagger2/oauth2/app.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +""" +Basic example of a resource server +""" + +import connexion + + +def get_secret(user) -> str: + return f"You are: {user}" + + +if __name__ == "__main__": + app = connexion.FlaskApp(__name__) + app.add_api("app.yaml") + app.run(port=8080) diff --git a/examples/swagger2/oauth2/app.yaml b/examples/swagger2/oauth2/app.yaml new file mode 100644 index 0000000..6d01a3a --- /dev/null +++ b/examples/swagger2/oauth2/app.yaml @@ -0,0 +1,30 @@ +swagger: "2.0" + +info: + title: OAuth Example + version: "1.0" + +paths: + /secret: + get: + summary: Return secret string + operationId: app.get_secret + responses: + 200: + description: secret response + schema: + type: string + security: + # enable authentication and require the "uid" scope for this endpoint + - oauth2: ['uid'] + +securityDefinitions: + oauth2: + type: oauth2 + flow: implicit + authorizationUrl: https://example.com/oauth2/dialog + # the token info URL is hardcoded for our mock_tokeninfo.py script + # you can also pass it as an environment variable TOKENINFO_URL + x-tokenInfoUrl: http://localhost:7979/tokeninfo + scopes: + uid: Unique identifier of the user accessing the service. diff --git a/examples/swagger2/oauth2/mock_tokeninfo.py b/examples/swagger2/oauth2/mock_tokeninfo.py new file mode 100755 index 0000000..0348446 --- /dev/null +++ b/examples/swagger2/oauth2/mock_tokeninfo.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +""" +Mock OAuth2 token info +""" + +import connexion +from connexion import request + +# our hardcoded mock "Bearer" access tokens +TOKENS = {"123": "jdoe", "456": "rms"} + + +def get_tokeninfo() -> dict: + try: + _, access_token = request.headers["Authorization"].split() + except Exception: + access_token = "" + + uid = TOKENS.get(access_token) + + if not uid: + return "No such token", 401 + + return {"uid": uid, "scope": ["uid"]} + + +if __name__ == "__main__": + app = connexion.FlaskApp(__name__) + app.add_api("mock_tokeninfo.yaml") + app.run(port=7979) diff --git a/examples/swagger2/oauth2/mock_tokeninfo.yaml b/examples/swagger2/oauth2/mock_tokeninfo.yaml new file mode 100644 index 0000000..9f2eb9f --- /dev/null +++ b/examples/swagger2/oauth2/mock_tokeninfo.yaml @@ -0,0 +1,23 @@ +swagger: "2.0" + +info: + title: Mock OAuth Token Info + version: "1.0" + +paths: + /tokeninfo: + get: + summary: OAuth2 token info + operationId: mock_tokeninfo.get_tokeninfo + responses: + 200: + description: Token info object + schema: + type: object + properties: + uid: + type: string + scope: + type: array + items: + type: string diff --git a/examples/swagger2/oauth2_local_tokeninfo/README.rst b/examples/swagger2/oauth2_local_tokeninfo/README.rst new file mode 100644 index 0000000..d1f4bcb --- /dev/null +++ b/examples/swagger2/oauth2_local_tokeninfo/README.rst @@ -0,0 +1,24 @@ +=============================== +OAuth2 Local Validation Example +=============================== + +This example demonstrates how to implement a resource server with Connexion. +The app will lookup OAuth2 Bearer tokens in a static map. + +Running: + +.. code-block:: bash + + $ sudo pip3 install --upgrade connexion # install Connexion from PyPI + $ ./app.py + +Now open your browser and go to http://localhost:8080/ui/ to see the Swagger UI. + +You can use the hardcoded tokens to request the endpoint: + +.. code-block:: bash + + $ curl http://localhost:8080/secret # missing authentication + $ curl -H 'Authorization: Bearer 123' http://localhost:8080/secret + $ curl -H 'Authorization: Bearer 456' http://localhost:8080/secret + diff --git a/examples/swagger2/oauth2_local_tokeninfo/app.py b/examples/swagger2/oauth2_local_tokeninfo/app.py new file mode 100755 index 0000000..3c0568c --- /dev/null +++ b/examples/swagger2/oauth2_local_tokeninfo/app.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +""" +Basic example of a resource server +""" + +import connexion + +# our hardcoded mock "Bearer" access tokens +TOKENS = {"123": "jdoe", "456": "rms"} + + +def get_secret(user) -> str: + return f"You are: {user}" + + +def token_info(access_token) -> dict: + uid = TOKENS.get(access_token) + if not uid: + return None + return {"uid": uid, "scope": ["uid"]} + + +if __name__ == "__main__": + app = connexion.FlaskApp(__name__) + app.add_api("app.yaml") + app.run(port=8080) diff --git a/examples/swagger2/oauth2_local_tokeninfo/app.yaml b/examples/swagger2/oauth2_local_tokeninfo/app.yaml new file mode 100644 index 0000000..67b45dd --- /dev/null +++ b/examples/swagger2/oauth2_local_tokeninfo/app.yaml @@ -0,0 +1,28 @@ +swagger: "2.0" + +info: + title: OAuth Example + version: "1.0" + +paths: + /secret: + get: + summary: Return secret string + operationId: app.get_secret + responses: + 200: + description: secret response + schema: + type: string + security: + # enable authentication and require the "uid" scope for this endpoint + - oauth2: ['uid'] + +securityDefinitions: + oauth2: + type: oauth2 + flow: implicit + authorizationUrl: https://example.com/oauth2/dialog + x-tokenInfoFunc: app.token_info + scopes: + uid: Unique identifier of the user accessing the service. diff --git a/firetail/auditor.py b/firetail/auditor.py index 8c7f0dd..9316580 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -59,16 +59,16 @@ def __init__(self, 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'pointsecFormat': { + 'firetailFormat': { 'format': '{"additional_field": "value"}', 'validate': False } }, 'handlers': { - 'pointsec': { - 'class': 'firetail.handlers.PointsecHandler', + 'firetail': { + 'class': 'firetail.handlers.FiretailHandler', 'level': 'DEBUG', - 'formatter': 'pointsecFormat', + 'formatter': 'firetailFormat', 'token': self.token, 'logs_drain_timeout': 5, 'url': self.url, @@ -80,7 +80,7 @@ def __init__(self, 'loggers': { '': { 'level': 'DEBUG', - 'handlers': ['pointsec'], + 'handlers': ['firetail'], 'propagate': True } } @@ -133,9 +133,9 @@ def create(self, response, token, scrub_headers=None, debug=False): self.scrub_headers = scrub_headers self.token = token if not self.logger: - self.LOGGING['handlers']['pointsec']['token'] = token + self.LOGGING['handlers']['firetail']['token'] = token logging.config.dictConfig(self.LOGGING) - self.logger = logging.getLogger('pointsecLogger') + self.logger = logging.getLogger('firetailLogger') payload = { "version": "1.1", diff --git a/firetail/exceptions.py b/firetail/exceptions.py index 6d42717..a8fd1cd 100644 --- a/firetail/exceptions.py +++ b/firetail/exceptions.py @@ -14,7 +14,7 @@ class FiretailException(Exception): pass -class PointsecException(Exception): +class FiretailException(Exception): pass diff --git a/firetail/flusher.py b/firetail/flusher.py index b358cf8..8e56ed8 100644 --- a/firetail/flusher.py +++ b/firetail/flusher.py @@ -2,7 +2,7 @@ import logging -class PointsecFlusher(logging.Logger): +class FiretailFlusher(logging.Logger): def __init__(self, logger): self.logger = logger diff --git a/firetail/handlers.py b/firetail/handlers.py index d3b158a..1934a6f 100644 --- a/firetail/handlers.py +++ b/firetail/handlers.py @@ -9,11 +9,11 @@ import sys import traceback -from .exceptions import (AuthenticationProblem, PointsecException, +from .exceptions import (AuthenticationProblem, FiretailException, ResolverProblem) from .logger import get_stdout_logger from .operations.secure import SecureOperation -from .sender import PointsecSender +from .sender import FiretailSender logger = logging.getLogger('firetail.handlers') @@ -97,13 +97,13 @@ def get_path_parameter_types(self): return {} -class PointsecHandler(logging.Handler): +class FiretailHandler(logging.Handler): def __init__(self, api_key, token, url, - pointsec_type="python", + firetail_type="python", logs_drain_timeout=3, debug=False, backup_logs=True, @@ -112,11 +112,11 @@ def __init__(self, retry_timeout=2): if not token: - raise PointsecException('pointsec.io Token must be provided') + raise FiretailException('firetail Token must be provided') - self.pointsec_type = pointsec_type + self.firetail_type = firetail_type - self.pointsec_sender = PointsecSender( + self.firetail_sender = FiretailSender( token=token, url=url, api_key=api_key, @@ -129,7 +129,7 @@ def __init__(self, logging.Handler.__init__(self) def __del__(self): - del self.pointsec_sender + del self.firetail_sender def extra_fields(self, message): @@ -158,10 +158,10 @@ def extra_fields(self, message): return extra_fields def flush(self): - self.pointsec_sender.flush() + self.firetail_sender.flush() def format(self, record): - message = super(PointsecHandler, self).format(record) + message = super(FiretailHandler, self).format(record) try: return json.loads(message) except (TypeError, ValueError): @@ -180,7 +180,7 @@ def format_message(self, message): # 'line_number': message.lineno, # 'path_name': message.pathname, # 'log_level': message.levelname, - # 'type': self.pointsec_type, + # 'type': self.firetail_type, # 'message': message.getMessage(), # '@timestamp': timestamp # } @@ -202,4 +202,4 @@ def emit(self, record): self.stdout_logger = get_stdout_logger(False) # self.stdout_logger.info(record.getMessage()) if 'ignore' not in message: - self.pointsec_sender.append(message) + self.firetail_sender.append(message) diff --git a/firetail/sender.py b/firetail/sender.py index edce031..b6fa342 100644 --- a/firetail/sender.py +++ b/firetail/sender.py @@ -1,4 +1,4 @@ -# This class is responsible for handling all asynchronous pointsec.io's +# This class is responsible for handling all asynchronous firetail's # communication import json import logging as loger4 @@ -25,12 +25,12 @@ def backup_logs(logs, logger): timestamp = datetime.now().strftime('%d%m%Y-%H%M%S') logger.info( - 'Backing up your logs to pointsec-failures-%s.txt', timestamp) - with open('pointsec-failures-{}.txt'.format(timestamp), 'a') as f: + 'Backing up your logs to firetail-failures-%s.txt', timestamp) + with open('firetail-failures-{}.txt'.format(timestamp), 'a') as f: f.writelines('\n'.join(logs)) -class PointsecSender: +class FiretailSender: def __init__(self, token, api_key, @@ -68,7 +68,7 @@ def __del__(self): def _initialize_sending_thread(self): self.sending_thread = Thread(target=self._drain_queue) self.sending_thread.daemon = False - self.sending_thread.name = 'pointsec-sending-thread' + self.sending_thread.name = 'firetail-sending-thread' self.sending_thread.start() def append(self, logs_message): @@ -97,7 +97,7 @@ def _drain_queue(self): self._flush_queue() except Exception as e: self.stdout_logger.debug( - 'Unexpected exception while draining queue to pointsec.io, ' + 'Unexpected exception while draining queue to firetail, ' 'swallowing. Exception: %s', e) if not last_try: @@ -109,7 +109,7 @@ def _flush_queue(self): while not self.queue.empty(): logs_list = self._get_messages_up_to_max_allowed_size() self.stdout_logger.debug( - 'Starting to drain %s logs to pointsec.io', len(logs_list)) + 'Starting to drain %s logs to firetail', len(logs_list)) # Not configurable from the outside sleep_between_retries = self.retry_timeout @@ -131,7 +131,7 @@ def _flush_queue(self): if response.status_code != 200: if response.status_code == 400: self.stdout_logger.debug( - 'Got 400 code from pointsec.io. This means that ' + 'Got 400 code from firetail. This means that ' 'some of your logs are too big, or badly ' 'formatted. response: %s', response.text) should_backup_to_disk = False @@ -140,13 +140,13 @@ def _flush_queue(self): if response.status_code == 401: self.stdout_logger.debug( - 'You are not authorized with pointsec.io! Token ' + 'You are not authorized with firetail! Token ' 'OK? dropping logs...') should_backup_to_disk = False break else: self.stdout_logger.debug( - 'Got %s while sending logs to pointsec.io, ' + 'Got %s while sending logs to firetail, ' 'Try (%s/%s). Response: %s', response.status_code, current_try + 1, @@ -156,12 +156,12 @@ def _flush_queue(self): else: self.stdout_logger.debug( 'Successfully sent bulk of %s logs to ' - 'pointsec.io!', len(logs_list)) + 'firetail', len(logs_list)) should_backup_to_disk = False break except Exception as e: self.stdout_logger.warning( - 'Got exception while sending logs to pointsec.io, ' + 'Got exception while sending logs to firetail, ' 'Try (%s/%s). Message: %s', current_try + 1, self.number_of_retries, e) should_retry = True @@ -172,7 +172,7 @@ def _flush_queue(self): if should_backup_to_disk and self.backup_logs: # Write to file self.stdout_logger.error( - 'Could not send logs to pointsec.io after %s tries, ' + 'Could not send logs to firetail after %s tries, ' 'backing up to local file system', self.number_of_retries) backup_logs(logs_list, self.stdout_logger) diff --git a/setup.py b/setup.py index cc72109..97a4796 100644 --- a/setup.py +++ b/setup.py @@ -92,8 +92,8 @@ def readme(): description='Firetail - API first applications with OpenAPI/Swagger and Flask', long_description=readme(), # long_description_content_type="text/x-rst", - author='point security inc', - url='https://github.com/PointSec-io/pointsecio', + author='Point Security Inc DBA FireTail (TM)', + url='https://github.com/FireTail-io/firetail-py-lib', keywords='openapi oai swagger rest api oauth flask microservice framework', license='LGPLv3', setup_requires=['flake8'], From efb3274350953f806bae245e19a806a1984f3988 Mon Sep 17 00:00:00 2001 From: Santhosh Shanmugam Date: Mon, 25 Jul 2022 09:50:03 +0100 Subject: [PATCH 03/10] Removed cloud logger changes overlap from rebranding branch and reset auditor back --- firetail/auditor.py | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 9316580..1a0fae7 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -10,22 +10,9 @@ from .logger import get_stdout_logger -DEFAULT_OPTIONS = dict(origins='*', - methods=['GET', 'HEAD', 'POST', 'OPTIONS', 'PUT', 'PATCH', 'DELETE'], - allow_headers='*', - expose_headers=None, - supports_credentials=False, - max_age=None, - send_wildcard=False, - automatic_options=True, - vary_header=True, - resources=r'/*', - intercept_exceptions=True, - always_send=True) -class cloud_logger(object): +class auditor: def __init__(self, - app, url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds', debug=False, @@ -59,16 +46,16 @@ def __init__(self, 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'firetailFormat': { + 'pointsecFormat': { 'format': '{"additional_field": "value"}', 'validate': False } }, 'handlers': { - 'firetail': { - 'class': 'firetail.handlers.FiretailHandler', + 'pointsec': { + 'class': 'firetail.handlers.PointsecHandler', 'level': 'DEBUG', - 'formatter': 'firetailFormat', + 'formatter': 'pointsecFormat', 'token': self.token, 'logs_drain_timeout': 5, 'url': self.url, @@ -80,18 +67,11 @@ def __init__(self, 'loggers': { '': { 'level': 'DEBUG', - 'handlers': ['firetail'], + 'handlers': ['pointsec'], 'propagate': True } } } - if app: - self.init_app(app, token) - - def init_app(self, app, token): - create_after_request = make_after_request_function(self,token) - app.after_request(create_after_request) - def set_token(self, token_secret): self.token = token_secret @@ -133,9 +113,9 @@ def create(self, response, token, scrub_headers=None, debug=False): self.scrub_headers = scrub_headers self.token = token if not self.logger: - self.LOGGING['handlers']['firetail']['token'] = token + self.LOGGING['handlers']['pointsec']['token'] = token logging.config.dictConfig(self.LOGGING) - self.logger = logging.getLogger('firetailLogger') + self.logger = logging.getLogger('pointsecLogger') payload = { "version": "1.1", @@ -170,9 +150,5 @@ def create(self, response, token, scrub_headers=None, debug=False): pass return payload -def make_after_request_function(cl,token): - def logs_after_request(resp): - cl.create(resp, token) - return resp - return logs_after_request +request_auditor = auditor() From 14dc244f30075d7ef61672ddbd07683bb91a23b2 Mon Sep 17 00:00:00 2001 From: Santhosh Shanmugam Date: Mon, 25 Jul 2022 10:15:23 +0100 Subject: [PATCH 04/10] Rebranded pointsec to firetail in auditor file after removing cloud logger changes, removed duplicate FiretailException class definition from exceptions.py --- firetail/auditor.py | 14 +++++++------- firetail/exceptions.py | 3 --- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 1a0fae7..7589c64 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -46,16 +46,16 @@ def __init__(self, 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'pointsecFormat': { + 'firetailFormat': { 'format': '{"additional_field": "value"}', 'validate': False } }, 'handlers': { - 'pointsec': { - 'class': 'firetail.handlers.PointsecHandler', + 'firetail': { + 'class': 'firetail.handlers.FiretailHandler', 'level': 'DEBUG', - 'formatter': 'pointsecFormat', + 'formatter': 'firetailFormat', 'token': self.token, 'logs_drain_timeout': 5, 'url': self.url, @@ -67,7 +67,7 @@ def __init__(self, 'loggers': { '': { 'level': 'DEBUG', - 'handlers': ['pointsec'], + 'handlers': ['firetail'], 'propagate': True } } @@ -113,9 +113,9 @@ def create(self, response, token, scrub_headers=None, debug=False): self.scrub_headers = scrub_headers self.token = token if not self.logger: - self.LOGGING['handlers']['pointsec']['token'] = token + self.LOGGING['handlers']['firetail']['token'] = token logging.config.dictConfig(self.LOGGING) - self.logger = logging.getLogger('pointsecLogger') + self.logger = logging.getLogger('firetailLogger') payload = { "version": "1.1", diff --git a/firetail/exceptions.py b/firetail/exceptions.py index a8fd1cd..f3c5cbe 100644 --- a/firetail/exceptions.py +++ b/firetail/exceptions.py @@ -14,9 +14,6 @@ class FiretailException(Exception): pass -class FiretailException(Exception): - pass - class ProblemException(FiretailException): def __init__(self, status=400, title=None, detail=None, type=None, From 8ae5a3050089466fe01e9e9c40e7b9432269864d Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Mon, 25 Jul 2022 15:46:07 +0300 Subject: [PATCH 05/10] Update pipeline.yml added fail-fast: false to debug pipeline failure --- .github/workflows/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index d6fff38..061ecea 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -11,6 +11,7 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] + fail-fast: false steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From bcabe9d1d04215259cd4a5ae6c820a9639f298bf Mon Sep 17 00:00:00 2001 From: Santhosh Shanmugam Date: Mon, 25 Jul 2022 13:57:29 +0100 Subject: [PATCH 06/10] Fixed flake8 issue in exceptions.py --- .github/workflows/pipeline.yml | 1 - firetail/exceptions.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 061ecea..d6fff38 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -11,7 +11,6 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] - fail-fast: false steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/firetail/exceptions.py b/firetail/exceptions.py index f3c5cbe..a5509c3 100644 --- a/firetail/exceptions.py +++ b/firetail/exceptions.py @@ -14,7 +14,6 @@ class FiretailException(Exception): pass - class ProblemException(FiretailException): def __init__(self, status=400, title=None, detail=None, type=None, instance=None, headers=None, ext=None): From ee487118c989667500936251da3a8ade2b72676c Mon Sep 17 00:00:00 2001 From: Riley Priddle Date: Tue, 26 Jul 2022 11:17:29 +0300 Subject: [PATCH 07/10] rebranded to firetail examples --- examples/swagger2/oauth2/app.py | 4 ++-- examples/swagger2/oauth2/mock_tokeninfo.py | 6 +++--- examples/swagger2/oauth2_local_tokeninfo/app.py | 4 ++-- tests/api/test_parameters.py | 7 ------- 4 files changed, 7 insertions(+), 14 deletions(-) mode change 100755 => 100644 examples/swagger2/oauth2/app.py mode change 100755 => 100644 examples/swagger2/oauth2/mock_tokeninfo.py mode change 100755 => 100644 examples/swagger2/oauth2_local_tokeninfo/app.py diff --git a/examples/swagger2/oauth2/app.py b/examples/swagger2/oauth2/app.py old mode 100755 new mode 100644 index 7450616..0bd36a8 --- a/examples/swagger2/oauth2/app.py +++ b/examples/swagger2/oauth2/app.py @@ -3,7 +3,7 @@ Basic example of a resource server """ -import connexion +import firetail def get_secret(user) -> str: @@ -11,6 +11,6 @@ def get_secret(user) -> str: if __name__ == "__main__": - app = connexion.FlaskApp(__name__) + app = firetail.FlaskApp(__name__) app.add_api("app.yaml") app.run(port=8080) diff --git a/examples/swagger2/oauth2/mock_tokeninfo.py b/examples/swagger2/oauth2/mock_tokeninfo.py old mode 100755 new mode 100644 index 0348446..ef1ff70 --- a/examples/swagger2/oauth2/mock_tokeninfo.py +++ b/examples/swagger2/oauth2/mock_tokeninfo.py @@ -3,8 +3,8 @@ Mock OAuth2 token info """ -import connexion -from connexion import request +import firetail +from firetail import request # our hardcoded mock "Bearer" access tokens TOKENS = {"123": "jdoe", "456": "rms"} @@ -25,6 +25,6 @@ def get_tokeninfo() -> dict: if __name__ == "__main__": - app = connexion.FlaskApp(__name__) + app = firetail.FlaskApp(__name__) app.add_api("mock_tokeninfo.yaml") app.run(port=7979) diff --git a/examples/swagger2/oauth2_local_tokeninfo/app.py b/examples/swagger2/oauth2_local_tokeninfo/app.py old mode 100755 new mode 100644 index 3c0568c..aa8171b --- a/examples/swagger2/oauth2_local_tokeninfo/app.py +++ b/examples/swagger2/oauth2_local_tokeninfo/app.py @@ -3,7 +3,7 @@ Basic example of a resource server """ -import connexion +import firetail # our hardcoded mock "Bearer" access tokens TOKENS = {"123": "jdoe", "456": "rms"} @@ -21,6 +21,6 @@ def token_info(access_token) -> dict: if __name__ == "__main__": - app = connexion.FlaskApp(__name__) + app = firetail.FlaskApp(__name__) app.add_api("app.yaml") app.run(port=8080) diff --git a/tests/api/test_parameters.py b/tests/api/test_parameters.py index fe86ae0..3c0a059 100644 --- a/tests/api/test_parameters.py +++ b/tests/api/test_parameters.py @@ -193,13 +193,6 @@ def test_path_parameter_somefloat(simple_app, arg, result): assert resp.data.decode('utf-8', 'replace') == f'"{result}"\n' -def test_path_parameter_somefloat__bad(simple_app): - # non-float values will not match Flask route - app_client = simple_app.app.test_client() - resp = app_client.get('/v1.0/test-float-path/123,45') # type: flask.Response - assert resp.status_code == 404 - - def test_default_param(strict_app): app_client = strict_app.app.test_client() resp = app_client.get('/v1.0/test-default-query-parameter') From 092e6c7700db3dffe2dce2862cf6a0841368060b Mon Sep 17 00:00:00 2001 From: Riley Priddle Date: Tue, 26 Jul 2022 11:19:05 +0300 Subject: [PATCH 08/10] Updated license link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a4e1519..8555670 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ Firetail :alt: Python Versions .. image:: https://img.shields.io/pypi/l/firetail.svg - :target: https://github.com/FireTail-io/firetail-py-lib/blob/main/LICENSE.txt + :target: https://raw.githubusercontent.com/FireTail-io/firetail-py-lib/main/LICENSE.txt :alt: License Firetail (fork of Connexion_) is a framework that automagically handles HTTP requests based on `OpenAPI Specification`_ From ed64095d4da6d5b03b19ac98a231b4a6acfb5b82 Mon Sep 17 00:00:00 2001 From: Riley Priddle Date: Tue, 26 Jul 2022 13:59:32 +0300 Subject: [PATCH 09/10] removed existing auditor updated top add hooks on execution time removed print ignore expired jwt tokens --- firetail/auditor.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 7589c64..16a9fc0 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -3,16 +3,18 @@ import json import logging import logging.config +import time import jwt import requests -from flask import request +from flask import g, request from .logger import get_stdout_logger -class auditor: +class cloud_logger(object): def __init__(self, + app, url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds', debug=False, @@ -72,6 +74,14 @@ def __init__(self, } } } + if app: + self.init_app(app, token) + + def init_app(self, app, token): + create_before_request = make_before_request_function() + app.before_request(create_before_request) + create_after_request = make_after_request_function(self, token) + app.after_request(create_after_request) def set_token(self, token_secret): self.token = token_secret @@ -97,7 +107,7 @@ def clean_pii(self, payload): if self.oauth and self.enrich_oauth: try: - jwt_decoded = jwt.decode(self.auth_token, options={"verify_signature": False}) + jwt_decoded = jwt.decode(self.auth_token, options={"verify_signature": False, "verify_exp": False}) except jwt.exceptions.DecodeError: self.oauth = False if self.oauth: @@ -106,7 +116,7 @@ def clean_pii(self, payload): payload['oauth']['email'] = jwt_decoded['email'] return payload - def create(self, response, token, scrub_headers=None, debug=False): + def create(self, response, token, diff=-1, scrub_headers=None, debug=False): if debug: self.stdout_logger = get_stdout_logger(True) if scrub_headers and isinstance(scrub_headers, list): @@ -120,6 +130,7 @@ def create(self, response, token, scrub_headers=None, debug=False): payload = { "version": "1.1", "dateCreated": int((datetime.datetime.utcnow()).timestamp() * 1000), + "execution_time": diff, "req": { "url": request.base_url, "headers": dict(request.headers), @@ -144,11 +155,24 @@ def create(self, response, token, scrub_headers=None, debug=False): try: if self.token: - print(json.dumps(self.clean_pii(payload))) self.logger.info(json.dumps(self.clean_pii(payload))) except TypeError: pass return payload -request_auditor = auditor() +def make_after_request_function(cl, token): + def logs_after_request(resp): + diff = time.time() - g.start + time_diff = diff * 1000 + print(round(time_diff, 2)) + cl.create(resp, token, round(time_diff, 2)) + return resp + return logs_after_request + + +def make_before_request_function(): + def logs_before_request(): + g.start = time.time() + + return logs_before_request From dfca9960692e403423d0d50ab8f0467571221867 Mon Sep 17 00:00:00 2001 From: Riley Priddle Date: Tue, 26 Jul 2022 15:18:22 +0300 Subject: [PATCH 10/10] added support for custom backend --- firetail/auditor.py | 11 +++++++---- firetail/handlers.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 16a9fc0..21ca928 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -18,6 +18,7 @@ def __init__(self, url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds', debug=False, + custom_backend=False, token=None, backup_logs=True, network_timeout=10.0, @@ -29,6 +30,7 @@ def __init__(self, ): self.api_key = api_key self.startThread = True + self.custom_backend = custom_backend self.requests_session = requests.Session() self.url = url self.token = token @@ -59,6 +61,7 @@ def __init__(self, 'level': 'DEBUG', 'formatter': 'firetailFormat', 'token': self.token, + 'custom_backend': self.custom_backend, 'logs_drain_timeout': 5, 'url': self.url, 'api_key': self.api_key, @@ -132,11 +135,12 @@ def create(self, response, token, diff=-1, scrub_headers=None, debug=False): "dateCreated": int((datetime.datetime.utcnow()).timestamp() * 1000), "execution_time": diff, "req": { + "httpProtocol": request.environ.get('SERVER_PROTOCOL', "HTTP/1.1"), "url": request.base_url, "headers": dict(request.headers), "path": request.path, "method": request.method, - "oPath": request.url_rule.rule if request.url_rule is not None else "", + "oPath": request.url_rule.rule if request.url_rule is not None else request.path, "fPath": request.full_path, "args": dict(request.args), "ip": request.remote_addr, @@ -152,9 +156,9 @@ def create(self, response, token, diff=-1, scrub_headers=None, debug=False): "content_type": response.content_type } } - + print(request.environ) try: - if self.token: + if self.token or self.custom_backend: self.logger.info(json.dumps(self.clean_pii(payload))) except TypeError: pass @@ -165,7 +169,6 @@ def make_after_request_function(cl, token): def logs_after_request(resp): diff = time.time() - g.start time_diff = diff * 1000 - print(round(time_diff, 2)) cl.create(resp, token, round(time_diff, 2)) return resp return logs_after_request diff --git a/firetail/handlers.py b/firetail/handlers.py index 1934a6f..ea21122 100644 --- a/firetail/handlers.py +++ b/firetail/handlers.py @@ -103,6 +103,7 @@ def __init__(self, api_key, token, url, + custom_backend=False, firetail_type="python", logs_drain_timeout=3, debug=False, @@ -111,7 +112,7 @@ def __init__(self, retries_no=4, retry_timeout=2): - if not token: + if not token and not custom_backend: raise FiretailException('firetail Token must be provided') self.firetail_type = firetail_type