From 4586ff3f4fa477daa6e0bee6c6e9e0602105c8d2 Mon Sep 17 00:00:00 2001 From: Konrad Scherer Date: Thu, 8 Mar 2018 14:52:05 -0500 Subject: [PATCH] WR specific configs and workflow for running on lpd-web Signed-off-by: Konrad Scherer --- .gitignore | 9 ++ Makefile | 148 ++++++++++++++++++ db_init.sql | 4 + db_reinit.sql | 4 + layerindex/management/__init__.py | 0 layerindex/management/commands/__init__.py | 0 .../management/commands/clone_branch.py | 34 ++++ settings.py | 42 ++--- urls.py | 2 +- 9 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 Makefile create mode 100644 db_init.sql create mode 100644 db_reinit.sql create mode 100644 layerindex/management/__init__.py create mode 100644 layerindex/management/commands/__init__.py create mode 100644 layerindex/management/commands/clone_branch.py diff --git a/.gitignore b/.gitignore index edfc2ac..39389cc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,13 @@ *.db3 *.swp .pytest_cache +.#* venv/ +.venv/ +.check +layerindex_db.sqlite3 +layerindex_db.sqlite3-journal +layerindex/tmp +tmp +bitbake.lock +layerindex/fixtures/* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..53d7655 --- /dev/null +++ b/Makefile @@ -0,0 +1,148 @@ +SHELL = /bin/bash #requires bash +VENV = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.venv +DEPS = $(wildcard *.py) +GET_PIP = $(VENV)/bin/get-pip.py +PIP = $(VENV)/bin/pip3 +DEBS = python3-dev libmysqlclient-dev + +.PHONY: local_instance setup clean test help update logged_update db_init db_reinit import images backup + +.DEFAULT_GOAL := local_instance + +help: + @echo "Make options for layerindex development" + @echo + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' + +# Use get-pip.py to avoid requiring installation of ensurepip package +$(VENV): .check + type python3 >/dev/null 2>&1 || { echo >&2 "Python3 required. Aborting."; exit 1; }; \ + test -d $(VENV) || python3 -m venv --without-pip $(VENV); \ + touch $(VENV); \ + wget -O $(GET_PIP) https://bootstrap.pypa.io/get-pip.py; \ + $(VENV)/bin/python3 $(GET_PIP) --ignore-installed; \ + $(PIP) install --upgrade --ignore-installed requests; \ + $(PIP) install --ignore-installed -r requirements.txt; \ + $(PIP) install --ignore-installed gunicorn; + +setup: $(VENV) ## Install all python dependencies in virtualenv. + +system: ## Convenience target for installing system libraries on Ubuntu/Debian + sudo apt-get install $(DEBS) + +.check: ## Verify system libraries are installed + @echo "Verifying system library installation" + @if [ -e /usr/bin/dpkg ]; then \ + for deb in $(DEBS); do \ + dpkg -L $$deb > /dev/null 2>&1; \ + if [ "$$?" != "0" ]; then \ + echo "Package $$deb must be installed. Run 'make system'."; \ + exit 1; \ + fi; \ + done; \ + else \ + echo "WARNING: unable to verify system libraries installed"; \ + fi; \ + touch .check + +clean: ## Delete virtualenv and all build directories + rm -rf $(VENV) .check + +test: ## Run tests + $(VENV)/bin/python3 setup.py test + +local_instance: layerindex_db.sqlite3 ## Run local django server with sqlitedb + $(VENV)/bin/python3 manage.py runserver 0.0.0.0:8000 + +layerindex_db.sqlite3: $(VENV) + $(VENV)/bin/python3 manage.py migrate + +update: $(VENV) ## Trigger update of layerindex database + . $(VENV)/bin/activate; ./layerindex/update.py --debug + +branch_update: $(VENV) ## Trigger update of a specific branch in layerindex database and redirect to $HOME/log +ifndef BRANCH + $(error BRANCH required ) +endif + $(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log) + mkdir -p $(HOME)/log; \ + . $(VENV)/bin/activate; ./layerindex/update.py --branch $(BRANCH) --debug > $(LOG) 2>&1; \ + echo $$? > $(VENV)/.$@_code.tmp ; \ + gzip $(LOG); \ + exit $$(cat $(VENV)/.$@_code.tmp) + +branch_full_update: $(VENV) ## Trigger full_reload of a specific branch in layerindex database and redirect to $HOME/log +ifndef BRANCH + $(error BRANCH required ) +endif + $(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log) + . $(VENV)/bin/activate; ./layerindex/update.py --branch $(BRANCH) --fullreload --debug > $(LOG) 2>&1; \ + echo $$? > $(VENV)/.$@_code.tmp ; \ + gzip $(LOG); \ + exit $$(cat $(VENV)/.$@_code.tmp) + +layer_full_update: $(VENV) ## Trigger full_reload of a specific layer and branch in layerindex database and redirect to $HOME/log +ifndef BRANCH + $(error BRANCH required ) +endif +ifndef LAYER + $(error LAYER required ) +endif + $(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log) + . $(VENV)/bin/activate; ./layerindex/update.py --layer $(LAYER) --branch $(BRANCH) --fullreload --debug > $(LOG) 2>&1; \ + echo $$? > $(VENV)/.$@_code.tmp ; \ + gzip $(LOG); \ + exit $$(cat $(VENV)/.$@_code.tmp) + +fullreload: $(VENV) ## Trigger update with full reload of layerindex database + . $(VENV)/bin/activate; ./layerindex/update.py --fullreload --debug + +migrate: $(VENV) ## Apply any database migrations to the mysql database +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + $(VENV)/bin/python3 manage.py migrate --settings $(SETTINGS) + +db_init: $(VENV) ## Create database and oelayer user with permissions +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + mysql -u root -p < db_init.sql; \ + $(VENV)/bin/python3 manage.py syncdb --settings $(SETTINGS); + +db_reinit: $(VENV) ## Drop and recreate database. Assume oelayer user already exists. +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + mysql -u root -p < db_reinit.sql; \ + $(VENV)/bin/python3 manage.py syncdb --settings $(SETTINGS); + +backup: $(VENV) ## Generate a dump of the entire db as a backup strategy +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + $(eval BACKUP=$(HOME)/db-backup/wrl-layerindex.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).json) + mkdir -p $(HOME)/db-backup; \ + $(VENV)/bin/python3 manage.py dumpdata --settings $(SETTINGS) --exclude=contenttypes --exclude=auth.Permission --exclude=corsheaders --exclude=reversion.version --exclude=reversion.revision --exclude=captcha.captchastore --exclude=sessions.session --exclude=layerindex.layerupdate > $(BACKUP); \ + gzip $(HOME)/db-backup/*.json; \ + ln -sfr $(BACKUP).gz $(HOME)/db-backup/latest.json.gz + +clone_branch: $(VENV) ## Clone an existing branch with a new name and description +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + $(VENV)/bin/python3 manage.py clone_branch --settings $(SETTINGS) \ + --branch "$(BRANCH)" --name "$(NAME)" --description "$(DESCRIPTION)" + +import: $(VENV) ## Load layerindex data exported from different layerindex +ifndef IMPORT + $(error IMPORT required ) +endif +ifndef SETTINGS + $(eval SETTINGS=settings) +endif + $(VENV)/bin/python3 manage.py loaddata --settings $(SETTINGS) $(IMPORT); \ + $(VENV)/bin/python3 manage.py migrate --settings $(SETTINGS); + +restart_gunicorn: ## Production instance uses gunicorn. Send HUP signal to trigger reload + /sbin/status layerindex-web | cut -d' ' -f 4 | xargs -r kill -HUP diff --git a/db_init.sql b/db_init.sql new file mode 100644 index 0000000..ff299af --- /dev/null +++ b/db_init.sql @@ -0,0 +1,4 @@ +CREATE DATABASE layerindex CHARACTER SET UTF8; +CREATE USER oelayer@localhost IDENTIFIED BY 'oelayer'; +GRANT ALL PRIVILEGES ON layerindex.* TO oelayer@localhost; +FLUSH PRIVILEGES; diff --git a/db_reinit.sql b/db_reinit.sql new file mode 100644 index 0000000..0f05eca --- /dev/null +++ b/db_reinit.sql @@ -0,0 +1,4 @@ +DROP DATABASE layerindex; +CREATE DATABASE layerindex CHARACTER SET UTF8; +GRANT ALL PRIVILEGES ON layerindex.* TO oelayer@localhost; +FLUSH PRIVILEGES; diff --git a/layerindex/management/__init__.py b/layerindex/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/layerindex/management/commands/__init__.py b/layerindex/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/layerindex/management/commands/clone_branch.py b/layerindex/management/commands/clone_branch.py new file mode 100644 index 0000000..3159de2 --- /dev/null +++ b/layerindex/management/commands/clone_branch.py @@ -0,0 +1,34 @@ +"""Clone a branch +Django does not support cloning directly but documents the best way to do it: +https://docs.djangoproject.com/en/1.8/topics/db/queries/#copying-model-instances +""" + +from django.core.management.base import BaseCommand, CommandError +from layerindex.models import Branch + +class Command(BaseCommand): + help = 'Clone an existing branch and give it a new name' + + def add_arguments(self, parser): + parser.add_argument('--branch', action='store', dest='branch', + required=True, help='The branch to clone') + parser.add_argument('--bitbake_branch', action='store', dest='bitbake_branch', + required=False, help='The branch to clone') + parser.add_argument('--name', action='store', dest='name', + required=True, help='The name of the cloned branch') + parser.add_argument('--description', action='store', dest='description', + required=True, help='The description of the cloned branch') + + def handle(self, *args, **options): + branch = Branch.objects.get(name=options['branch']) + branch.name = options['name'] + branch.short_description = options['description'] + + # By default the bitbake branch will match the branch name + branch.bitbake_branch = options.get('bitbake_branch') + if branch.bitbake_branch is None: + branch.bitbake_branch = options['name'] + + # Setting primary key to none will give the clone a new pk on save + branch.pk = None + branch.save() diff --git a/settings.py b/settings.py index a456eb7..aa7c201 100644 --- a/settings.py +++ b/settings.py @@ -6,19 +6,23 @@ import os DEBUG = False +TEMPLATE_DEBUG = DEBUG +ALLOWED_HOSTS = ['*'] ADMINS = ( - # ('Your Name', 'your_email@example.com'), + ('Konrad Scherer', 'Konrad.Scherer@windriver.com'), + ('Mark Hatle', 'Mark.Hatle@windriver.com'), + ('Robert', 'liezhi.yang@windriver.com'), ) MANAGERS = ADMINS DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': '', # Or path to database file if using sqlite3 (full path recommended). - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. + 'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': 'layerindex', # Or path to database file if using sqlite3 (full path recommended). + 'USER': 'oelayer', # Not used with sqlite3. + 'PASSWORD': 'oelayer', # Not used with sqlite3. 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 'PORT': '', # Set to empty string for default. Not used with sqlite3. } @@ -31,13 +35,13 @@ # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. -TIME_ZONE = 'Europe/London' +TIME_ZONE = 'US/Pacific' # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = 'en-us' -SITE_ID = 1 +SITE_ID = 2 # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. @@ -67,12 +71,12 @@ # URL prefix for static files. # Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' +STATIC_URL = '/layerindex/static/' # URL prefix for admin static files -- CSS, JavaScript and images. # Make sure to use a trailing slash. # Examples: "http://foo.com/static/admin/", "/static/admin/". -ADMIN_MEDIA_PREFIX = '/static/admin/' +ADMIN_MEDIA_PREFIX = '/layerindex/static/admin/' # Additional locations of static files STATICFILES_DIRS = ( @@ -90,7 +94,7 @@ ) # Make this unique, and don't share it with anybody. -SECRET_KEY = '' +SECRET_KEY = 'b9480d70-80d1-11e6-a35d-5f30d8e86a21' MIDDLEWARE_CLASSES = ( 'corsheaders.middleware.CorsMiddleware', @@ -198,41 +202,41 @@ # Registration settings ACCOUNT_ACTIVATION_DAYS = 2 -EMAIL_HOST = 'smtp.example.com' -DEFAULT_FROM_EMAIL = 'noreply@example.com' +EMAIL_HOST = 'prod-webmail.wrs.com' +DEFAULT_FROM_EMAIL = 'Konrad.Scherer@windriver.com' LOGIN_REDIRECT_URL = '/layerindex' # Full path to directory where layers should be fetched into by the update script -LAYER_FETCH_DIR = "" +LAYER_FETCH_DIR = "/home/oelayer/layerindex" # Base temporary directory in which to create a directory in which to run BitBake TEMP_BASE_DIR = "/tmp" # Fetch URL of the BitBake repository for the update script -BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake" +BITBAKE_REPO_URL = "git://lxgit.wrs.com/bitbake" # Core layer to be used by the update script for basic BitBake configuration CORE_LAYER_NAME = "openembedded-core" # Update records older than this number of days will be deleted every update -UPDATE_PURGE_DAYS = 30 +UPDATE_PURGE_DAYS = 10 # Remove layer dependencies that are not specified in conf/layer.conf -REMOVE_LAYER_DEPENDENCIES = False +REMOVE_LAYER_DEPENDENCIES = True # Always use https:// for review URLs in emails (since it may be redirected to # the login page) FORCE_REVIEW_HTTPS = False # Settings for layer submission feature -SUBMIT_EMAIL_FROM = 'noreply@example.com' -SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission' +SUBMIT_EMAIL_FROM = 'Konrad.Scherer@windriver.com' +SUBMIT_EMAIL_SUBJECT = 'Wind River Linux Layerindex layer submission' # Send email to maintainer(s) when their layer is published SEND_PUBLISH_EMAIL = True # RabbitMQ settings -RABBIT_BROKER = 'amqp://' +RABBIT_BROKER = 'amqp://admin:mypass@localhost:5672/vhost' RABBIT_BACKEND = 'rpc://' # Used for fetching repo diff --git a/urls.py b/urls.py index 52c518a..3540a5f 100644 --- a/urls.py +++ b/urls.py @@ -26,5 +26,5 @@ ] urlpatterns += [ - url(r'.*', RedirectView.as_view(url='/layerindex/', permanent=False)), + url(r'.*', RedirectView.as_view(url='layerindex/', permanent=False)), ]