diff --git a/.gitignore b/.gitignore index 7d281ceb..9ae69fe1 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ test.db* # don't check in NCBI data ncbi/ + +# compiled static files +static/ diff --git a/Makefile b/Makefile index 768660c4..af72f92e 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ clean-pyc: clean-js: rm -rf src/edge/static/edge/lib/* - rm -vf src/edge/static/edge/edge{.css,.html,.js,.min.js,_jst.js} + rm -rvf src/edge/static/edge/assets/* # Testing diff --git a/README.rst b/README.rst index a21acff9..533da5dc 100644 --- a/README.rst +++ b/README.rst @@ -34,27 +34,37 @@ Try it using Docker ------------------- * Use ``docker-compose``: -To start edge server: +To start the edge server: + :: - + docker-compose up -Then check it out in your browser: http://localhost:9000/edge/#/genomes -To import an genome using: + +Then check it out in your browser: http://localhost:9000/edge/#/genomes . + +To import a genome, use: + :: + docker-compose run edge python src/manage.py import_gff 'Saccharomyces cerevisiae' example/sc_s288c.gff -To run a shell inside the edge container + +To run a shell inside the Edge container: + :: + docker-compose run --rm edge bash * Alternatively, you can use the ``Makefile``: To start edge server: + :: make start-ext -To import an genome as an example: +To import a genome as an example: + :: make add-s288c-ext @@ -66,11 +76,10 @@ trying to rebuild the image. The ``Makefile`` holds all the commands necessary for managing the server and database, both in usage and development. Run ``make`` without arguments to see a list of commmands. -The Docker environment is defined in ``docker-compose.yml``. Ese the ``edge`` service for your +The Docker environment is defined in ``docker-compose.yml``. Use the ``edge`` service for your commands. -Of course, any of these ``make`` targets can be run directly from a shell inside a container from -either service: +Any of these ``make`` targets can be run directly from a shell inside a container: :: @@ -125,10 +134,9 @@ Then, to set up the edge BLAST db, from the ``src`` subdirectory, Editing data ------------ -You can edit genome and fragment metadata, such as name, notes, circular -attributes, from the Django admin. Create a Django admin superuser, (see the ``superuser`` make -target), then set your browser to the ``/admin/`` endpoint of wherever you are running your dev -server. +You can edit genome and fragment metadata, such as name, notes, circular attributes, from the Django +admin. Create a Django admin superuser, (see the ``superuser`` make target), then set your browser +to the ``/admin/`` endpoint of wherever you are running your dev server. Deploying to production @@ -138,7 +146,8 @@ Do *not* use the Dockerfile as-is for production, or the ``make run`` task. Djan command is not meant to run a production server. Instead, you'll need to spin up a production WSGI server and run the Django projct with that, with your own settings. In this situation, it's better to simply install the ``edge-genome`` python package on your deployed system and add it to your -deployment Django server's ``installed_apps`` setting. +deployment Django server's ``installed_apps`` setting. The package is designed so that, when built, +it already contains all of the javascript assets compiled in their final state. Development, testing, and package release @@ -147,6 +156,14 @@ Development, testing, and package release When developing locally, you can run tests in the controlled environment of the docker container from your local machine with ``make test-ext.`` +Note that edge uses webassets_ for compilation of static assets. These assets are not automatically +compiled (because the integration of that with Django is flaky). Instead, compile assets after +cahnging them with ``make build_assets``. + +Static dependencies are managed with Bower_. (Eventually to be replaced with npm_/webpack_). +Dependencies are downloaded before the python package is built so python package consumers already +have all required javascript. + Edge is versioned semantically. Continuous integration is done automatically on all branches through Travis CI, and tagged commits to master are automatically released to PyPI. To release a new version, bump the version number with the appropriate severity of the changes (major, minor, or patch), and @@ -158,3 +175,8 @@ push the resulting tagged commits to the GitHub remote repo: you@localhost:edge$ git push --tags origin master If you cannot push to master directly, do the same thing on a new branch and submit a pull request. + +.. _webassets: https://webassets.readthedocs.io/ +.. _Bower: https://bower.io/ +.. _npm: https://npmjs.org/ +.. _webpack: https://webpack.js.org/ diff --git a/docker-compose.yml b/docker-compose.yml index 9caa4027..1e6301d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: MYSQL_DATABASE: edge_dev volumes: - edge-db-data:/var/lib/mysql - + rabbit: image: rabbitmq diff --git a/requirements-core.txt b/requirements-core.txt index 986bcbc7..5e527a71 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -10,7 +10,7 @@ flower==0.9.1 bcbio-gff == 0.4 # Web assets -django_assets == 0.8 +django_assets == 0.12 jsmin == 2.0.9 numpy == 1.12.1 diff --git a/setup.py b/setup.py index 59a81173..eecb61e2 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,28 @@ from __future__ import print_function import os -from subprocess import check_output, STDOUT +import sys +from subprocess import ( + CalledProcessError, + STDOUT, + check_output, +) from setuptools import setup, find_packages, Command from setuptools.command.sdist import sdist -class BowerInstallCommand(Command): +class SubproccessCommand(Command): + def _run_with_output(self, *args, **kwargs): + try: + print(check_output(*args, **kwargs)) + except CalledProcessError as e: + print(e.output, file=sys.stderr) + + raise e + + +class BowerInstallCommand(SubproccessCommand): description = 'run bower commands [install by default]' user_options = [ ('bower-command=', 'c', 'Bower command to run. Defaults to "install".'), @@ -35,10 +50,10 @@ def run(self): cmd.append('-p') if self.allow_root: cmd.append('--allow-root') - print(check_output(cmd, stderr=STDOUT)) + self._run_with_output(cmd, stderr=STDOUT) -class BuildAssetsCommand(Command): +class BuildAssetsCommand(SubproccessCommand): description = 'build django assets' user_options = [] @@ -50,7 +65,8 @@ def finalize_options(self): def run(self): src_dir = os.path.join(os.path.dirname(__file__), 'src') - print(check_output(['python', 'manage.py', 'assets', 'build'], cwd=src_dir, stderr=STDOUT)) + self._run_with_output(['python', 'manage.py', 'assets', 'build'], + cwd=src_dir, stderr=STDOUT) class SdistCommandWithJS(sdist): @@ -109,7 +125,7 @@ def run(self): zip_safe=True, setup_requires=[ - 'django_assets', + 'django_assets >= 0.12', ], install_requires=[ 'django ~= 1.11.6', diff --git a/src/.gitignore b/src/.gitignore index 080600fb..0da89660 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,7 +1,4 @@ db.sqlite3* test.db* .webassets-cache -edge/static/edge/edge.css -edge/static/edge/edge.js -edge/static/edge/edge.min.js -edge/static/edge/edge_jst.js +edge/static/edge/assets/ diff --git a/src/edge/assets.py b/src/edge/assets.py index dbccd9d6..22521b3a 100644 --- a/src/edge/assets.py +++ b/src/edge/assets.py @@ -1,13 +1,17 @@ from django_assets import Bundle, register -js = Bundle('edge/src/js/*.js', filters='jsmin', output='edge/static/edge/edge.min.js') +js = Bundle('edge/static/edge/src/js/*.js', + output='edge/static/edge/assets/edge.js') register('edge_js', js) -js = Bundle('edge/src/js/*.js', output='edge/static/edge/edge.js') -register('edge', js) +jsmin = Bundle(js, filters='jsmin', + output='edge/static/edge/assets/edge.min.js') +register('edge_jsmin', jsmin) -css = Bundle('edge/src/css/app.css', output='edge/static/edge/edge.css') +css = Bundle('edge/static/edge/src/css/app.css', + output='edge/static/edge/assets/edge.css') register('edge_css', css) -jst = Bundle('edge/src/partials/*.html', filters='jst', output='edge/static/edge/edge_jst.js') +jst = Bundle('edge/static/edge/src/partials/*.html', filters='jst', + output='edge/static/edge/assets/edge_jst.js') register('edge_jst', jst) diff --git a/src/edge/templates/edge/edge.html b/src/edge/templates/edge/edge.html index 4281f38b..8fa224f9 100644 --- a/src/edge/templates/edge/edge.html +++ b/src/edge/templates/edge/edge.html @@ -18,10 +18,10 @@ - - - - + + + + diff --git a/src/edge/urls.py b/src/edge/urls.py index c848311a..0e0c352d 100644 --- a/src/edge/urls.py +++ b/src/edge/urls.py @@ -26,7 +26,7 @@ # we have to add a redirect from the static page to the /ui/ URL, for any # browser that have the redirect memorized. - url(r'^/?$', TemplateView.as_view(template_name='edge/edge.html'), name='edge-ui'), + url(r'^$', TemplateView.as_view(template_name='edge/edge.html'), name='edge-ui'), url(r'^ui/?$', TemplateView.as_view(template_name='edge/edge.html')), # APIs diff --git a/src/server/settings.py b/src/server/settings.py index 2b0268f1..d92997cb 100644 --- a/src/server/settings.py +++ b/src/server/settings.py @@ -8,8 +8,10 @@ https://docs.djangoproject.com/en/1.6/ref/settings/ """ -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import sys import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) # Quick-start development settings - unsuitable for production @@ -21,11 +23,9 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -TEMPLATE_DEBUG = True ALLOWED_HOSTS = ['*'] -import sys TESTING = sys.argv[1:2] == ['test'] # for Django Celery @@ -35,7 +35,25 @@ CELERY_SEND_TASK_SENT_EVENT = True if TESTING: - CELERY_ALWAYS_EAGER = True # skip the daemon + CELERY_ALWAYS_EAGER = True # skip the daemon + +# Tempalte loaders +TEMPLATES = [{ + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'APP_DIRS': True, + 'OPTIONS': { + 'debug': DEBUG, + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.request', + 'django.contrib.messages.context_processors.messages', + ], + }, +}] # Application definition @@ -57,7 +75,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', - #'django.middleware.csrf.CsrfViewMiddleware', + # 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', @@ -78,12 +96,14 @@ }, 'mysql': { 'ENGINE': 'django.db.backends.mysql', - 'OPTIONS': { "init_command": "SET storage_engine=INNODB;" }, - 'HOST': os.getenv("DB_HOST", ""), - 'PORT': "", - 'NAME': os.getenv("DB_NAME", ""), - 'USER': os.getenv("DB_USER", ""), - 'PASSWORD': os.getenv('DB_PASSWORD', ""), + 'OPTIONS': { + 'init_command': "SET storage_engine=INNODB; SET sql_mode='STRICT_TRANS_TABLES';" + }, + 'HOST': os.getenv('DB_HOST', ''), + 'PORT': '', + 'NAME': os.getenv('DB_NAME', ''), + 'USER': os.getenv('DB_USER', ''), + 'PASSWORD': os.getenv('DB_PASSWORD', ''), 'ATOMIC_REQUESTS': True, } } @@ -111,10 +131,23 @@ # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.6/howto/static-files/ STATIC_URL = '/static/' +# Files compiled with django's assets go in the static directory in the project root +STATIC_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'static') + +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + # XXX We do not automatically use django-assets finder here, as it is buggy. + # Instead, we use staticfiles for everything, and manually manage compilation of webassests + # via `setup.py build_assets`, or the `manage.py assets build` +) + +# All django-assets paths in apps' assets.py files are relative to BASE_DIR +ASSETS_ROOT = BASE_DIR + LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -130,7 +163,7 @@ } }, 'loggers': { - #'django.db.backends': { 'level': 'DEBUG', 'handlers': ['console'], }, + # 'django.db.backends': { 'level': 'DEBUG', 'handlers': ['console'], }, }, } @@ -146,6 +179,6 @@ NOSE_ARGS = [ - '--with-coverage', - '--cover-package=edge', + '--with-coverage', + '--cover-package=edge', ]