WR specific configs and workflow for running on lpd-web

Signed-off-by: Konrad Scherer <Konrad.Scherer@windriver.com>

.gitignore

@@ -2,4 +2,13 @@
 *.db3
 *.swp
 .pytest_cache
+.#*
 venv/
+.venv/
+.check
+layerindex_db.sqlite3
+layerindex_db.sqlite3-journal
+layerindex/tmp
+tmp
+bitbake.lock
+layerindex/fixtures/*

Makefile

@@ -0,0 +1,148 @@
+SHELL = /bin/bash #requires bash
+VENV = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/.venv
+DEPS = $(wildcard *.py)
+GET_PIP = $(VENV)/bin/get-pip.py
+PIP = $(VENV)/bin/pip3
+DEBS = python3-dev libmysqlclient-dev
+
+.PHONY: local_instance setup clean test help update logged_update db_init db_reinit import images backup
+
+.DEFAULT_GOAL := local_instance
+
+help:
+	@echo "Make options for layerindex development"
+	@echo
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}'
+
+# Use get-pip.py to avoid requiring installation of ensurepip package
+$(VENV): .check
+	type python3 >/dev/null 2>&1 || { echo >&2 "Python3 required. Aborting."; exit 1; }; \
+	test -d $(VENV) || python3 -m venv --without-pip $(VENV); \
+	touch $(VENV); \
+	wget -O $(GET_PIP) https://bootstrap.pypa.io/get-pip.py; \
+	$(VENV)/bin/python3 $(GET_PIP) --ignore-installed; \
+	$(PIP) install --upgrade --ignore-installed requests; \
+	$(PIP) install --ignore-installed -r requirements.txt; \
+	$(PIP) install --ignore-installed gunicorn;
+
+setup: $(VENV) ## Install all python dependencies in virtualenv.
+
+system: ## Convenience target for installing system libraries on Ubuntu/Debian
+	sudo apt-get install $(DEBS)
+
+.check: ## Verify system libraries are installed
+	@echo "Verifying system library installation"
+	@if [ -e /usr/bin/dpkg ]; then \
+		for deb in $(DEBS); do \
+			dpkg -L $$deb > /dev/null 2>&1; \
+			if [ "$$?" != "0" ]; then \
+				echo "Package $$deb must be installed. Run 'make system'."; \
+				exit 1; \
+			fi; \
+		done; \
+	else \
+		echo "WARNING: unable to verify system libraries installed"; \
+	fi; \
+	touch .check
+
+clean: ## Delete virtualenv and all build directories
+	rm -rf $(VENV) .check
+
+test: ## Run tests
+	$(VENV)/bin/python3 setup.py test
+
+local_instance: layerindex_db.sqlite3 ## Run local django server with sqlitedb
+	$(VENV)/bin/python3 manage.py runserver 0.0.0.0:8000
+
+layerindex_db.sqlite3: $(VENV)
+	$(VENV)/bin/python3 manage.py migrate
+
+update: $(VENV) ## Trigger update of layerindex database
+	. $(VENV)/bin/activate; ./layerindex/update.py --debug
+
+branch_update: $(VENV) ## Trigger update of a specific branch in layerindex database and redirect to $HOME/log
+ifndef BRANCH
+	$(error BRANCH required )
+endif
+	$(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log)
+	mkdir -p $(HOME)/log; \
+	. $(VENV)/bin/activate; ./layerindex/update.py --branch $(BRANCH) --debug > $(LOG) 2>&1; \
+	echo $$? > $(VENV)/.$@_code.tmp ; \
+	gzip $(LOG); \
+	exit $$(cat $(VENV)/.$@_code.tmp)
+
+branch_full_update: $(VENV) ## Trigger full_reload of a specific branch in layerindex database and redirect to $HOME/log
+ifndef BRANCH
+	$(error BRANCH required )
+endif
+	$(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log)
+	. $(VENV)/bin/activate; ./layerindex/update.py --branch $(BRANCH) --fullreload --debug > $(LOG) 2>&1; \
+	echo $$? > $(VENV)/.$@_code.tmp ; \
+	gzip $(LOG); \
+	exit $$(cat $(VENV)/.$@_code.tmp)
+
+layer_full_update: $(VENV) ## Trigger full_reload of a specific layer and branch in layerindex database and redirect to $HOME/log
+ifndef BRANCH
+	$(error BRANCH required )
+endif
+ifndef LAYER
+	$(error LAYER required )
+endif
+	$(eval LOG=$(HOME)/log/layerindex-update.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).log)
+	. $(VENV)/bin/activate; ./layerindex/update.py --layer $(LAYER) --branch $(BRANCH) --fullreload --debug > $(LOG) 2>&1; \
+	echo $$? > $(VENV)/.$@_code.tmp ; \
+	gzip $(LOG); \
+	exit $$(cat $(VENV)/.$@_code.tmp)
+
+fullreload: $(VENV) ## Trigger update with full reload of layerindex database
+	. $(VENV)/bin/activate; ./layerindex/update.py --fullreload --debug
+
+migrate: $(VENV) ## Apply any database migrations to the mysql database
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	$(VENV)/bin/python3 manage.py migrate --settings $(SETTINGS)
+
+db_init: $(VENV) ## Create database and oelayer user with permissions
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	mysql -u root -p < db_init.sql; \
+	$(VENV)/bin/python3 manage.py syncdb --settings $(SETTINGS);
+
+db_reinit: $(VENV) ## Drop and recreate database. Assume oelayer user already exists.
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	mysql -u root -p < db_reinit.sql; \
+	$(VENV)/bin/python3 manage.py syncdb --settings $(SETTINGS);
+
+backup: $(VENV) ## Generate a dump of the entire db as a backup strategy
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	$(eval BACKUP=$(HOME)/db-backup/wrl-layerindex.$(shell date +\%Y_\%m_\%d-\%H\%M\%S).json)
+	mkdir -p $(HOME)/db-backup; \
+	$(VENV)/bin/python3 manage.py dumpdata --settings $(SETTINGS) --exclude=contenttypes --exclude=auth.Permission --exclude=corsheaders --exclude=reversion.version --exclude=reversion.revision --exclude=captcha.captchastore --exclude=sessions.session --exclude=layerindex.layerupdate > $(BACKUP); \
+	gzip $(HOME)/db-backup/*.json; \
+	ln -sfr $(BACKUP).gz $(HOME)/db-backup/latest.json.gz
+
+clone_branch: $(VENV) ## Clone an existing branch with a new name and description
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	$(VENV)/bin/python3 manage.py clone_branch --settings $(SETTINGS) \
+        --branch "$(BRANCH)" --name "$(NAME)" --description "$(DESCRIPTION)"
+
+import: $(VENV) ## Load layerindex data exported from different layerindex
+ifndef IMPORT
+	$(error IMPORT required )
+endif
+ifndef SETTINGS
+	$(eval SETTINGS=settings)
+endif
+	$(VENV)/bin/python3 manage.py loaddata --settings $(SETTINGS) $(IMPORT); \
+	$(VENV)/bin/python3 manage.py migrate --settings $(SETTINGS);
+
+restart_gunicorn: ## Production instance uses gunicorn. Send HUP signal to trigger reload
+	/sbin/status layerindex-web | cut -d' ' -f 4 | xargs -r kill -HUP

db_init.sql

@@ -0,0 +1,4 @@
+CREATE DATABASE layerindex CHARACTER SET UTF8;
+CREATE USER oelayer@localhost IDENTIFIED BY 'oelayer';
+GRANT ALL PRIVILEGES ON layerindex.* TO oelayer@localhost;
+FLUSH PRIVILEGES;

db_reinit.sql

@@ -0,0 +1,4 @@
+DROP DATABASE layerindex;
+CREATE DATABASE layerindex CHARACTER SET UTF8;
+GRANT ALL PRIVILEGES ON layerindex.* TO oelayer@localhost;
+FLUSH PRIVILEGES;

layerindex/management/commands/clone_branch.py

@@ -0,0 +1,34 @@
+"""Clone a branch
+Django does not support cloning directly but documents the best way to do it:
+https://docs.djangoproject.com/en/1.8/topics/db/queries/#copying-model-instances
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+from layerindex.models import Branch
+
+class Command(BaseCommand):
+    help = 'Clone an existing branch and give it a new name'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--branch', action='store', dest='branch',
+                            required=True, help='The branch to clone')
+        parser.add_argument('--bitbake_branch', action='store', dest='bitbake_branch',
+                            required=False, help='The branch to clone')
+        parser.add_argument('--name', action='store', dest='name',
+                            required=True, help='The name of the cloned branch')
+        parser.add_argument('--description', action='store', dest='description',
+                            required=True, help='The description of the cloned branch')
+
+    def handle(self, *args, **options):
+        branch = Branch.objects.get(name=options['branch'])
+        branch.name = options['name']
+        branch.short_description = options['description']
+
+        # By default the bitbake branch will match the branch name
+        branch.bitbake_branch = options.get('bitbake_branch')
+        if branch.bitbake_branch is None:
+            branch.bitbake_branch = options['name']
+
+        # Setting primary key to none will give the clone a new pk on save
+        branch.pk = None
+        branch.save()

settings.py

@@ -6,19 +6,23 @@
 import os
 
 DEBUG = False
+TEMPLATE_DEBUG = DEBUG
+ALLOWED_HOSTS = ['*']
 
 ADMINS = (
-    # ('Your Name', 'your_email@example.com'),
+    ('Konrad Scherer', 'Konrad.Scherer@windriver.com'),
+    ('Mark Hatle', 'Mark.Hatle@windriver.com'),
+    ('Robert', 'liezhi.yang@windriver.com'),
 )
 
 MANAGERS = ADMINS
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-        'NAME': '',                      # Or path to database file if using sqlite3 (full path recommended).
-        'USER': '',                      # Not used with sqlite3.
-        'PASSWORD': '',                  # Not used with sqlite3.
+        'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'layerindex',                 # Or path to database file if using sqlite3 (full path recommended).
+        'USER': 'oelayer',                    # Not used with sqlite3.
+        'PASSWORD': 'oelayer',                # Not used with sqlite3.
         'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
         'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
     }
@@ -31,13 +35,13 @@
 # timezone as the operating system.
 # If running in a Windows environment this must be set to the same as your
 # system time zone.
-TIME_ZONE = 'Europe/London'
+TIME_ZONE = 'US/Pacific'
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
 LANGUAGE_CODE = 'en-us'
 
-SITE_ID = 1
+SITE_ID = 2
 
 # If you set this to False, Django will make some optimizations so as not
 # to load the internationalization machinery.
@@ -67,12 +71,12 @@
 
 # URL prefix for static files.
 # Example: "http://media.lawrence.com/static/"
-STATIC_URL = '/static/'
+STATIC_URL = '/layerindex/static/'
 
 # URL prefix for admin static files -- CSS, JavaScript and images.
 # Make sure to use a trailing slash.
 # Examples: "http://foo.com/static/admin/", "/static/admin/".
-ADMIN_MEDIA_PREFIX = '/static/admin/'
+ADMIN_MEDIA_PREFIX = '/layerindex/static/admin/'
 
 # Additional locations of static files
 STATICFILES_DIRS = (
@@ -90,7 +94,7 @@
 )
 
 # Make this unique, and don't share it with anybody.
-SECRET_KEY = ''
+SECRET_KEY = 'b9480d70-80d1-11e6-a35d-5f30d8e86a21'
 
 MIDDLEWARE_CLASSES = (
     'corsheaders.middleware.CorsMiddleware',
@@ -198,41 +202,41 @@
 
 # Registration settings
 ACCOUNT_ACTIVATION_DAYS = 2
-EMAIL_HOST = 'smtp.example.com'
-DEFAULT_FROM_EMAIL = 'noreply@example.com'
+EMAIL_HOST = 'prod-webmail.wrs.com'
+DEFAULT_FROM_EMAIL = 'Konrad.Scherer@windriver.com'
 LOGIN_REDIRECT_URL = '/layerindex'
 
 # Full path to directory where layers should be fetched into by the update script
-LAYER_FETCH_DIR = ""
+LAYER_FETCH_DIR = "/home/oelayer/layerindex"
 
 # Base temporary directory in which to create a directory in which to run BitBake
 TEMP_BASE_DIR = "/tmp"
 
 # Fetch URL of the BitBake repository for the update script
-BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake"
+BITBAKE_REPO_URL = "git://lxgit.wrs.com/bitbake"
 
 # Core layer to be used by the update script for basic BitBake configuration
 CORE_LAYER_NAME = "openembedded-core"
 
 # Update records older than this number of days will be deleted every update
-UPDATE_PURGE_DAYS = 30
+UPDATE_PURGE_DAYS = 10
 
 # Remove layer dependencies that are not specified in conf/layer.conf
-REMOVE_LAYER_DEPENDENCIES = False
+REMOVE_LAYER_DEPENDENCIES = True
 
 # Always use https:// for review URLs in emails (since it may be redirected to
 # the login page)
 FORCE_REVIEW_HTTPS = False
 
 # Settings for layer submission feature
-SUBMIT_EMAIL_FROM = 'noreply@example.com'
-SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
+SUBMIT_EMAIL_FROM = 'Konrad.Scherer@windriver.com'
+SUBMIT_EMAIL_SUBJECT = 'Wind River Linux Layerindex layer submission'
 
 # Send email to maintainer(s) when their layer is published
 SEND_PUBLISH_EMAIL = True
 
 # RabbitMQ settings
-RABBIT_BROKER = 'amqp://'
+RABBIT_BROKER = 'amqp://admin:mypass@localhost:5672/vhost'
 RABBIT_BACKEND = 'rpc://'
 
 # Used for fetching repo

urls.py

@@ -26,5 +26,5 @@
     ]
 
 urlpatterns += [
-    url(r'.*', RedirectView.as_view(url='/layerindex/', permanent=False)),
+    url(r'.*', RedirectView.as_view(url='layerindex/', permanent=False)),
 ]