From 473fa6fe7877c67aa50e600d4526e94e00c468e0 Mon Sep 17 00:00:00 2001 From: Amit Tripathi Date: Thu, 6 Dec 2018 22:09:20 +0200 Subject: [PATCH] Added docker-compose setup & code to support it --- Dockerfile | 3 +- README.md | 47 ++++++++++++++++++++++++--- TODO.rst | 6 ++++ docker-compose.yml | 52 ++++++++++++++++++++++++++++++ pygmy/config/config.py | 5 +-- pygmy/config/pygmy.cfg | 20 +++++++----- pygmy/config/pygmy_test.cfg | 4 +-- pygmy/database/base.py | 64 ++++++++++++++++++++++++++++++++++--- pygmy/rest/shorturl.py | 3 -- pygmyui/restclient/base.py | 4 +++ pygmyui/restclient/pygmy.py | 8 +++++ pygmyui/utils.py | 13 ++++++-- requirements.txt | 3 +- 13 files changed, 204 insertions(+), 28 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index de0ce41..1b1a3ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ LABEL version='1.0.0' LABEL description='Pygmy(pygy.co) URL shortener' LABEL vendor="Amit Tripathi" -RUN apt update -y && apt install python3-pip -y +RUN apt update && apt install python3-pip -y +RUN mkdir /var/log/pygmy WORKDIR /pygmy ADD ./requirements.txt /pygmy/requirements.txt diff --git a/README.md b/README.md index 1c71ded..7320a4f 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Build Status](https://travis-ci.org/amitt001/pygmy.svg?branch=master)](https://travis-ci.org/amitt001/pygmy) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/Django.svg) [![PyPI license](https://img.shields.io/pypi/l/ansicolortags.svg)](https://pypi.python.org/pypi/ansicolortags/) +![Docker Pulls](https://img.shields.io/docker/pulls/amit19/pygmy.svg) [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/amit19) Live version of this project @ [https://pygy.co](https://pygy.co) @@ -26,6 +27,7 @@ If you would like to supprt the project, I can be contacted on my email of sourc - [Use MySQL](#use-mysql) - [Use Postgresql](#use-postgresql) - [Use SQLite](#use-sqlite) + - [Docker](#docker-1) - [Using Pygmy API](#using-pygmy-api) - [Create User:](#create-user) - [Shell Usage](#shell-usage) @@ -65,6 +67,7 @@ The architecture is very loosely coupled which allows custom integrations easily - DB: PostgreSQL/MySQL/SQLite - Others: SQLAlchmey, JWT - Docker +- Docker-compose ## Installation/Setup @@ -99,6 +102,10 @@ Note: ## DB Setup: +By default Pygmy uses SQLite but anoy of the SQLite, MySQL, PostgreSQL can be used. Configs are present on pygmy.cfg file in `pygmy/config` directory. + +Use db specific instruction below. Make sure to check and modify values in pygmy.cfg file according to your DB setup. + ### Use MySQL First install `pymysql`: @@ -112,8 +119,14 @@ Check correct port: Change below line in `pygmy/core/pygmy.cfg`: ``` +[database] engine: mysql -url: mysql+pymysql://root:root@127.0.0.1:3306/pygmy +url: {engine}://{user}:{password}@{host}:{port}/{db_name} +user: root +password: root +host: 127.0.0.1 +port: 3306 +db_name: pygmy ``` Enter MySQL URL @@ -124,9 +137,18 @@ Note: Better using Mysql with version > `5.6.5` to use default value of `CURRENT ### Use Postgresql -`pip install psycopg2` +Change below line in `pygmy/core/pygmy.cfg`: -`postgres://amit@127.0.0.1:5432/pygmy` +``` +[database] +engine: postgresql +url: {engine}://{user}:{password}@{host}:{port}/{db_name} +user: root +password: root +host: 127.0.0.1 +port: 5432 +db_name: pygmy +``` ### Use SQLite @@ -134,6 +156,23 @@ SQLite is natively supported in Python `sqlite:////var/lib/pygmy/pygmy.db` +``` +[database] +engine: sqlite3 +sqlite_data_dir: data +sqlite_db_file_name: pygmy.db +``` + +## Docker + +Docker image name: amit19/pygmy + +Docker image can be build by: `docker build -t amit19/pygmy .` + +Both Dockerfile and docker-compose file are preset at the root of the project. + +To use docker-compose you need to pass DB credentials in docker-compose file + ## Using Pygmy API ## Create User: @@ -263,7 +302,7 @@ See coverage report(Coverage is bad because the coverage for integration tests i Thanks [batarian71](https://github.com/batarian71) for providing the logo icon. ## Donations -If this project help you reduce time to develop, you can buy me a cup of coffee :) +If this project help you reduce time to develop, you can help me keep this project running by donating any amount you see fit :) [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/amit19) diff --git a/TODO.rst b/TODO.rst index 3312c8d..40ac390 100644 --- a/TODO.rst +++ b/TODO.rst @@ -2,6 +2,12 @@ TODO ==== +* Reduce image size from 550+MB to > 200MB + +* Complete docker compose setup + +* Deploy pygy.co demo website with docker compose + * For same netloc same url, forward stash same url * If url pattern doesn't match. Show a 404 page diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1056001 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3' + +services: + + database: + image: postgres:11 + restart: always + ports: + - "5433:5432" + volumes: + - pgdata:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: root + POSTGRES_USER: root + POSTGRES_DB: pygmy + + pygmy: + image: pygmy:develop + restart: always + build: . + # ports: + # - "9119:9119" + links: + - database + environment: + - DB_PASSWORD=root + - DB_USER=root + - DB_NAME=pygmy + - DB_HOST=database + - DB_PORT=5432 + volumes: + - .:/pygmy + command: gunicorn --log-file /var/log/pygmy/error_logs.log --access-logfile /var/log/pygmy/acclogs.log --log-level DEBUG --bind 0.0.0.0:9119 --workers 2 pygmy.rest.wsgi:app + depends_on: + - database + + pygmyui: + image: pygmy:develop + restart: always + build: . + ports: + - "8000:8000" + links: + - pygmy + environment: + - PYGMY_API_ADDRESS=pygmy + command: sh -c "cd pygmyui && gunicorn --log-file /var/log/pygmy/uierror_logs.log --access-logfile /var/log/pygmy/uiacclogs.log --bind 0.0.0.0:8000 --workers 2 pygmyui.wsgi && cd .." + depends_on: + - pygmy + +volumes: + pgdata: diff --git a/pygmy/config/config.py b/pygmy/config/config.py index 700e2a3..59fdf76 100644 --- a/pygmy/config/config.py +++ b/pygmy/config/config.py @@ -55,6 +55,7 @@ def initialize(self): if self.database['engine'] == 'sqlite3': root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - sqlite_path = root_dir + '/' + self.database['data_dir'] + '/' + self.database['file_name'] + data_dir = self.database.get('sqlite_data_dir') or 'data' + file_name = self.database.get('sqlite_db_file_name') or 'pygmy.db' + sqlite_path = root_dir + '/' + data_dir + '/' + file_name self.database['url'] = 'sqlite:///{}'.format(sqlite_path) - diff --git a/pygmy/config/pygmy.cfg b/pygmy/config/pygmy.cfg index e895e6c..bf7081e 100644 --- a/pygmy/config/pygmy.cfg +++ b/pygmy/config/pygmy.cfg @@ -8,15 +8,19 @@ short_url = 127.0.0.1 short_url_schema = http:// [database] -file_name: pygmy.db +# Engone can be sqlite3, postgresql, mysql engine: sqlite3 -data_dir = data -# url: postgres://amit@127.0.0.1:5432/pygmy -# mysql+pymysql://root:root@127.0.0.1:3306/pygmy -user: -password: -host: -port: +# Only required if engine is sqlite3 +sqlite_data_dir: data +sqlite_db_file_name: pygmy.db +url: {engine}://{user}:{password}@{host}:{port}/{db_name} +# NOTE: Modify these fields according to your DB +user: root +password: root +host: 127.0.0.1 +# Mysql default port is 3306 +port: 5432 +db_name: pygmy [auth] refresh_secret = _))_((*REFRESH)(*_)+_ diff --git a/pygmy/config/pygmy_test.cfg b/pygmy/config/pygmy_test.cfg index 280e380..39f8a9a 100644 --- a/pygmy/config/pygmy_test.cfg +++ b/pygmy/config/pygmy_test.cfg @@ -8,9 +8,9 @@ short_url = 127.0.0.1 short_url_schema = http:// [database] -file_name: pygmy_test.db +sqlite_db_file_name: pygmy_test.db engine: sqlite3 -data_dir = data +sqlite_data_dir = data # url: postgres://amit@127.0.0.1:5432/pygmy # sqlite:////var/lib/pygmy/pygmy.db # mysql+pymysql://root:root@127.0.0.1:3306/pygmy diff --git a/pygmy/database/base.py b/pygmy/database/base.py index c53f566..16613ed 100644 --- a/pygmy/database/base.py +++ b/pygmy/database/base.py @@ -1,3 +1,5 @@ +import os + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base @@ -11,7 +13,7 @@ class BaseDatabase: def __init__(self): self.engine = None - self.url = None + self._db_url = None self.store = None def _prepare(self, url): @@ -24,11 +26,63 @@ def commit(self): def abort(self): self.store.rollback() + # TODO: Probs, should be done in config + @property + def db_url(self): + """Gets the url from config file pygmy.cfg and then look up for + following enviorment variable. If found, replcases the values + + - DB_HOST + - DB_PORT + - DB_USER + - DB_PASS + - DB_DBNAME + + The reason for two ways to get DB url is support for both CLI + based runs and ducker-compose/kubernetes based runs + """ + if self._db_url is not None: + return self._db_url + + self._db_url = config.database['url'] + if config.database['engine'] == 'sqlite3': + return self._db_url + + # As in case of mysql we use pymysql, modify engine here + if config.database['engine'] == 'mysql': + engine = 'mysql+pymysql' + else: + engine = config.database['engine'] + + + # Get environment variables + host, port, user, password, db_name = ( + os.environ.get('DB_HOST'), os.environ.get('DB_PORT'), + os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'), + os.environ.get('DB_NAME')) + + if any([host, port, user, password, db_name]): + log.info('Replacing config value by environment variable') + + url_kw_params = { + 'engine': engine, + 'user': user or config.database['user'], + 'password': password or config.database['password'], + 'host': host or config.database['host'], + 'port': port or config.database['port'], + 'db_name': db_name or config.database['db_name'] + } + try: + self._db_url = self._db_url.format(**url_kw_params) + except KeyError as err: + # Raised if one of the config is not passed + raise KeyError('Key: {} not set in config file'.format(err)) + + def initialize(self, debug=False): - self.url = config.database['url'] - log.info('DB URL: {}'.format(self.url)) - self._prepare(self.url) - self.engine = create_engine(self.url, echo=debug) + log.info('DB URL: {}'.format(self.db_url)) + self._prepare(self.db_url) + self.engine = create_engine(self.db_url, echo=debug) session = sessionmaker(bind=self.engine) self.store = session() self.store.commit() diff --git a/pygmy/rest/shorturl.py b/pygmy/rest/shorturl.py index fd616f9..bab9f23 100644 --- a/pygmy/rest/shorturl.py +++ b/pygmy/rest/shorturl.py @@ -73,9 +73,6 @@ class ShortURLApi(MethodView): def get(self): secret = request.headers.get('secret_key') short_url = request.args.get('url') - is_valid = validate_url(short_url) - if is_valid is False: - return jsonify(dict(error='Invalid URL.')), 400 try: long_url = unshorten(short_url, secret_key=secret, query_by_code=True, request=request) diff --git a/pygmyui/restclient/base.py b/pygmyui/restclient/base.py index 73d2838..9664d08 100644 --- a/pygmyui/restclient/base.py +++ b/pygmyui/restclient/base.py @@ -1,6 +1,7 @@ import os import sys import requests +import logging from functools import wraps @@ -21,6 +22,8 @@ FORBIDDEN = 403 RESOURCE_EXPIRED = 410 +logger = logging.getLogger(__name__) + def catch_connection_error(func): @wraps(func) @@ -108,6 +111,7 @@ def call(self, path, data=None, method=None, error_object = self.error_object_from_response(response) if error_object is not None: + logger.debug('Reveived error response: %s', response.text) raise error_object return response diff --git a/pygmyui/restclient/pygmy.py b/pygmyui/restclient/pygmy.py index a0225b5..9f5cfd8 100644 --- a/pygmyui/restclient/pygmy.py +++ b/pygmyui/restclient/pygmy.py @@ -18,6 +18,14 @@ class PygmyApiClient(Client): """Pygmy REST API wrapper class""" def __init__(self, rest_url, username, password, hostname, request=None): + """ + Arguments: + rest_url {[type]} -- Pygmy API url + username {[type]} -- api username + password {[type]} -- api password + hostname {[type]} -- the pygmy website/ui hostname. Used to create the short url + """ + self.name = username self.password = password self.rest_url = rest_url diff --git a/pygmyui/utils.py b/pygmyui/utils.py index da465d5..7227473 100644 --- a/pygmyui/utils.py +++ b/pygmyui/utils.py @@ -1,6 +1,9 @@ """A common util file to use methods across different django apps""" +import os +import logging from restclient.pygmy import PygmyApiClient + from urllib.parse import urlparse @@ -13,7 +16,13 @@ def make_url(url_address): def pygmy_client_object(config, request): rest_user = config.PYGMY_API_USER rest_pass = config.PYGMY_API_PASSWORD - rest_url = make_url(config.PYGMY_API_ADDRESS) + pygmy_api_host, pygmy_api_port = urlparse(config.PYGMY_API_ADDRESS).netloc.split(':') + + # Check if PYGMY_API_ADDRESS enviornment varibale is set + if os.environ.get('PYGMY_API_ADDRESS'): + pygmy_api_host = os.environ.get('PYGMY_API_ADDRESS') + logging.info('Using environment variable PYGMY_API_ADDRESS. API URL: %s', pygmy_api_host) + + rest_url = make_url('http://{}:{}'.format(pygmy_api_host, pygmy_api_port)) hostname = config.HOSTNAME return PygmyApiClient(rest_url, rest_user, rest_pass, hostname, request) - diff --git a/requirements.txt b/requirements.txt index 41afd31..e925524 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ marshmallow==2.16.3 maxminddb==1.3.0 passlib==1.7.1 pluggy==0.6.0 +psycopg2==2.7.6.1 py==1.5.2 pycparser==2.18 PyJWT==1.5.3 @@ -27,4 +28,4 @@ requests==2.20.1 six==1.11.0 SQLAlchemy==1.2.2 urllib3==1.22 -Werkzeug==0.14.1 +Werkzeug==0.14.1 \ No newline at end of file