Skip to content
Permalink
Browse files

Per resource scheduling (#222)

* #132 first stab at healthcheck scheduler using daemon process

* #132 first stab at healthcheck scheduler using daemon process, fix tests

* #132 further progress on separate test freq+ runner daemon: Docker support

* #132 further progress on runner daemon: Docker support and docs

* #132 further progress on runner daemon: Docker support and docs - flake8 fix

* #132 make runner in webapp default, more Docker support and docs

* #132 make runner in webapp default, more Docker support and docs - flake8

* #132 fix Alembic upgrade error server_default with quotes

* #132 fix job for flushing runs (retention days format)

* #132 fix conveting bool env vars in config_site.py for Docker

* #132 extend end-time for ResourceLock to prevent missing obtainments

* #132 less  end-time for ResourceLock to prevent missing obtainments

* #132 less  end-time for ResourceLock to prevent missing obtainments

* SCRIPT_NAME must not be empty string or only slash

* SCRIPT_NAME may not be slash and minimal run frequency

* safeguard run_frequency lower than confiured minimum

* some docu changes for testing DockerHub webhook

* schedule first run at random time to prevent all_at_once

* refine scheduler and job parameterization for liveliness

* set VERSION to master one
  • Loading branch information...
justb4 authored and tomkralidis committed Nov 5, 2018
1 parent 4a02ed5 commit 1cc4b3efc56a154044adecf1f8b33962fe673efe
Showing with 1,185 additions and 502 deletions.
  1. +1 −1 .travis.yml
  2. +53 −52 Dockerfile
  3. +21 −42 GeoHealthCheck/app.py
  4. +5 −1 GeoHealthCheck/config_main.py
  5. +12 −7 GeoHealthCheck/factory.py
  6. +80 −15 GeoHealthCheck/healthcheck.py
  7. +35 −30 GeoHealthCheck/init.py
  8. +47 −0 GeoHealthCheck/migrations/versions/34531bfd7cab_.py
  9. +101 −89 GeoHealthCheck/models.py
  10. +4 −1 GeoHealthCheck/plugin.py
  11. +281 −0 GeoHealthCheck/scheduler.py
  12. +10 −0 GeoHealthCheck/templates/edit_resource.html
  13. +2 −2 GeoHealthCheck/templates/layout.html
  14. +6 −0 GeoHealthCheck/templates/resource.html
  15. +10 −0 GeoHealthCheck/translations/en/LC_MESSAGES/messages.po
  16. +10 −0 GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po
  17. +2 −2 GeoHealthCheck/util.py
  18. +22 −27 README.md
  19. +92 −105 docker/README.md
  20. +7 −0 docker/compose/README.md
  21. +65 −21 docker/compose/docker-compose.postgis.yml
  22. +33 −57 docker/compose/docker-compose.yml
  23. +7 −0 docker/compose/ghc-postgis.env
  24. +25 −0 docker/compose/ghc.env
  25. +15 −6 docker/config_site.py
  26. +3 −0 docker/docker-clean.sh
  27. +1 −1 docker/install-docker-ubuntu.sh
  28. +1 −1 docker/{ → scripts}/configure.sh
  29. 0 docker/{ → scripts}/cron-jobs-daily.sh
  30. +1 −1 docker/{ → scripts}/cron-jobs-hourly.sh
  31. +3 −2 docker/{ → scripts}/install.sh
  32. +17 −0 docker/scripts/run-runner.sh
  33. +16 −4 docker/{run.sh → scripts/run-web.sh}
  34. +23 −0 docker/scripts/set-timezone.sh
  35. BIN docs/_static/ghc-parts.jpg
  36. +45 −13 docs/admin.rst
  37. +92 −1 docs/architecture.rst
  38. +5 −7 docs/config.rst
  39. +1 −1 docs/index.rst
  40. +14 −9 docs/install.rst
  41. +1 −1 jobs.cron
  42. +13 −1 pavement.py
  43. +3 −2 requirements.txt
@@ -14,7 +14,7 @@ script:
- echo -e "admin\ntest\ntest\nyou@example.com\nyou@example.com" | python GeoHealthCheck/models.py create
- flake8
- python GeoHealthCheck/models.py load tests/data/fixtures.json y
- python GeoHealthCheck/models.py run
- python GeoHealthCheck/healthcheck.py
- python tests/run_tests.py
- cd docs && make html

@@ -4,81 +4,82 @@ FROM python:2.7.13-alpine
# FROM debian:jessie

# Credits to yjacolin for providing first versions
LABEL original_developer "yjacolin <yves.jacolin@camptocamp.com>"
LABEL maintainer "Just van den Broecke <justb4@gmail.com>"
LABEL original_developer="yjacolin <yves.jacolin@camptocamp.com>" \
maintainer="Just van den Broecke <justb4@gmail.com>"

# These are default values,
# Override when running container via docker(-compose)

# General ENV settings
ENV LC_ALL "en_US.UTF-8"
ENV LANG "en_US.UTF-8"
ENV LANGUAGE "en_US.UTF-8"

# GHC ENV settings
ENV ADMIN_NAME admin
ENV ADMIN_PWD admin
ENV ADMIN_EMAIL admin.istrator@mydomain.com
ENV SQLALCHEMY_DATABASE_URI 'sqlite:////GeoHealthCheck/DB/data.db'
ENV SECRET_KEY 'd544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342'
ENV GHC_RETENTION_DAYS 30
ENV GHC_RUN_FREQUENCY 'hourly'
ENV GHC_PROBE_HTTP_TIMEOUT_SECS 30
ENV GHC_SELF_REGISTER False
ENV GHC_NOTIFICATIONS False
ENV GHC_NOTIFICATIONS_VERBOSITY True
ENV GHC_WWW_LINK_EXCEPTION_CHECK False
ENV GHC_ADMIN_EMAIL 'you@example.com'
ENV GHC_NOTIFICATIONS_EMAIL 'you2@example.com,them@example.com'
ENV GHC_SITE_TITLE 'GeoHealthCheck'
ENV GHC_SITE_URL 'http://localhost'
ENV GHC_SMTP_SERVER None
ENV GHC_SMTP_PORT None
ENV GHC_SMTP_TLS False
ENV GHC_SMTP_SSL False
ENV GHC_SMTP_USERNAME None
ENV GHC_SMTP_PASSWORD None
ENV GHC_METADATA_CACHE_SECS 900

# WSGI server settings, assumed is gunicorn
ENV HOST 0.0.0.0
ENV PORT 80
ENV WSGI_WORKERS 4
ENV WSGI_WORKER_TIMEOUT 6000
ENV WSGI_WORKER_CLASS 'eventlet'

# GHC Core Plugins modules and/or classes, seldom needed to set:
# if not specified here or in Container environment
# all GHC built-in Plugins will be active.
ENV LC_ALL="en_US.UTF-8" \
LANG="en_US.UTF-8" \
LANGUAGE="en_US.UTF-8" \
\
\
# GHC ENV settings\
ADMIN_NAME=admin \
ADMIN_PWD=admin \
ADMIN_EMAIL=admin.istrator@mydomain.com \
SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \
SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \
GHC_PROBE_HTTP_TIMEOUT_SECS=30 \
GHC_MINIMAL_RUN_FREQUENCY_MINS=10 \
GHC_RETENTION_DAYS=30 \
GHC_SELF_REGISTER=False \
GHC_NOTIFICATIONS=False \
GHC_NOTIFICATIONS_VERBOSITY=True \
GHC_WWW_LINK_EXCEPTION_CHECK=False \
GHC_ADMIN_EMAIL='you@example.com' \
GHC_RUNNER_IN_WEBAPP=False \
GHC_LOG_LEVEL=30 \
GHC_LOG_FORMAT='%(asctime)s - %(name)s - %(levelname)s - %(message)s' \
GHC_NOTIFICATIONS_EMAIL='you2@example.com,them@example.com' \
GHC_SITE_TITLE='GeoHealthCheck' \
GHC_SITE_URL='http://localhost' \
GHC_SMTP_SERVER=None \
GHC_SMTP_PORT=None \
GHC_SMTP_TLS=False \
GHC_SMTP_SSL=False \
GHC_SMTP_USERNAME=None \
GHC_SMTP_PASSWORD=None \
GHC_METADATA_CACHE_SECS=900 \
\
# WSGI server settings, assumed is gunicorn \
HOST=0.0.0.0 \
PORT=80 \
WSGI_WORKERS=4 \
WSGI_WORKER_TIMEOUT=6000 \
WSGI_WORKER_CLASS='eventlet' \
\
# GHC Core Plugins modules and/or classes, seldom needed to set: \
# if not specified here or in Container environment \
# all GHC built-in Plugins will be active. \
#ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\
# GeoHealthCheck.plugins.probe.wms, ...., ...\
# GeoHealthCheck.plugins.check.checks'

# GHC User Plugins, best be overridden via Container environment
ENV GHC_USER_PLUGINS ''
# GeoHealthCheck.plugins.check.checks' \
\
# GHC User Plugins, best be overridden via Container environment \
GHC_USER_PLUGINS=''

RUN apk add --no-cache --virtual .build-deps gcc build-base linux-headers postgresql-dev \
&& apk add --no-cache bash vim postgresql-client \
&& apk add --no-cache bash postgresql-client tzdata openntpd \
&& pip install virtualenv \
&& rm -rf /var/cache/apk/* /tmp/* /var/tmp/*

# Add standard files and Add/override Plugins
# Alternative Entrypoints to run GHC jobs
# Override default Entrypoint with these on Containers
ADD docker/install.sh docker/configure.sh docker/run.sh \
docker/config_site.py docker/cron-jobs-daily.sh docker/cron-jobs-hourly.sh docker/plugins /
RUN chmod a+x /*.sh
ADD docker/scripts/*.sh docker/config_site.py docker/plugins /

# Add Source Code
ADD . /GeoHealthCheck

# Install and Remove build-related packages for smaller image size
RUN bash install.sh \
&& apk del .build-deps
RUN chmod a+x /*.sh && bash install.sh && apk del .build-deps

# For SQLite
VOLUME ["/GeoHealthCheck/DB/"]

EXPOSE ${PORT}

ENTRYPOINT /configure.sh && /run.sh
ENTRYPOINT /run-web.sh
@@ -32,7 +32,6 @@
import csv
import logging
import json
from datetime import datetime, timedelta
from StringIO import StringIO
from itertools import chain

@@ -44,7 +43,6 @@
from flask_migrate import Migrate

from __init__ import __version__
from healthcheck import sniff_test_resource, run_test_resource
from init import App
from enums import RESOURCE_TYPES
from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
@@ -74,6 +72,16 @@
('hr_HR', 'Croatian (Croatia)')
)

# Should GHC Runner be run within GHC webapp?
if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True:
LOGGER.info('Running GHC Scheduler in WebApp')
from scheduler import start_schedule

# Start scheduler
start_schedule()
else:
LOGGER.info('NOT Running GHC Scheduler in WebApp')


# commit or rollback shorthand
def db_commit():
@@ -122,35 +130,6 @@ def unauthorized_callback():
return redirect(url_for('login', lang=g.current_lang, next=url))


def next_page_refresh():
"""determines when to refresh webapp based on GHC_RUN_FREQUENCY"""

now = datetime.now()

frequency = CONFIG['GHC_RUN_FREQUENCY']

if frequency == 'hourly': # get next hour
now2 = now.replace(minute=0, second=0, microsecond=0)
refresh = timedelta(hours=1)
elif frequency == 'daily': # get next day
now2 = now.replace(hour=0, minute=0, second=0, microsecond=0)
refresh = timedelta(days=1)
elif frequency == ['weekly']: # get next day
now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
refresh = timedelta(weeks=1)
elif frequency == ['monthly']:
now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
refresh = timedelta(weeks=4)
elif frequency == ['yearly']: # get next day
now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
refresh = timedelta(weeks=52)

next_frequency = now2 + refresh
differ = next_frequency - now

return differ.seconds


@APP.template_filter('cssize_reliability')
def cssize_reliability(value, css_type=None):
"""returns CSS button class snippet based on score"""
@@ -162,11 +141,11 @@ def cssize_reliability(value, css_type=None):
score = 'danger'
panel = 'red'
elif (CONFIG['GHC_RELIABILITY_MATRIX']['orange']['min'] <= number <=
CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']):
CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']):
score = 'warning'
panel = 'yellow'
elif (CONFIG['GHC_RELIABILITY_MATRIX']['green']['min'] <= number <=
CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']):
CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']):
score = 'success'
panel = 'green'
else: # should never really get here
@@ -203,7 +182,6 @@ def context_processors():
tags = views.get_tag_counts()
return {
'app_version': __version__,
'next_page_refresh': next_page_refresh(),
'resource_types': RESOURCE_TYPES,
'resource_types_counts': rtc['counts'],
'resources_total': rtc['total'],
@@ -490,6 +468,7 @@ def add():
url = request.form['url'].strip()
resources_to_add = []

from healthcheck import sniff_test_resource, run_test_resource
sniffed_resources = sniff_test_resource(CONFIG, resource_type, url)

if not sniffed_resources:
@@ -557,7 +536,7 @@ def add():
param_vals = {}
for param in param_defs:
if param_defs[param]['value']:
param_vals[param] =\
param_vals[param] = \
param_defs[param]['value']
check_vars = CheckVars(
probe_to_add, check_class, param_vals)
@@ -641,8 +620,8 @@ def update(resource_identifier):

# Add ProbeVars anew each with optional CheckVars
for probe in value:
print('adding Probe class=%s parms=%s' %
(probe['probe_class'], str(probe)))
LOGGER.info('adding Probe class=%s parms=%s' %
(probe['probe_class'], str(probe)))
probe_vars = ProbeVars(resource, probe['probe_class'],
probe['parameters'])
for check in probe['checks']:
@@ -663,6 +642,9 @@ def update(resource_identifier):
elif getattr(resource, key) != resource_identifier_dict[key]:
# Update other resource attrs, mainly 'name'
setattr(resource, key, resource_identifier_dict[key])
min_run_freq = CONFIG['GHC_MINIMAL_RUN_FREQUENCY_MINS']
if int(resource.run_frequency) < min_run_freq:
resource.run_frequency = min_run_freq
update_counter += 1

# Always update geo-IP: maybe failure on creation or
@@ -699,6 +681,7 @@ def test(resource_identifier):
flash(gettext('Resource not found'), 'danger')
return redirect(request.referrer)

from healthcheck import run_test_resource
result = run_test_resource(
resource)

@@ -754,10 +737,6 @@ def delete(resource_identifier):
flash(gettext('Resource not found'), 'danger')
return redirect(url_for('home', lang=g.current_lang))

runs = Run.query.filter_by(resource_identifier=resource_identifier).all()

for run in runs:
DB.session.delete(run)
resource.clear_recipients()
DB.session.delete(resource)

@@ -908,7 +887,7 @@ def api_probes_avail(resource_type=None, resource_id=None):

if __name__ == '__main__': # run locally, for fun
import sys
logging.basicConfig()

HOST = '0.0.0.0'
PORT = 8000
if len(sys.argv) > 1:
@@ -38,8 +38,8 @@
SECRET_KEY = None

GHC_RETENTION_DAYS = 30
GHC_RUN_FREQUENCY = 'hourly'
GHC_PROBE_HTTP_TIMEOUT_SECS = 30
GHC_MINIMAL_RUN_FREQUENCY_MINS = 10
GHC_SELF_REGISTER = False
GHC_NOTIFICATIONS = False
GHC_NOTIFICATIONS_VERBOSITY = True
@@ -48,6 +48,10 @@
GHC_NOTIFICATIONS_EMAIL = ['you2@example.com']
GHC_SITE_TITLE = 'GeoHealthCheck Demonstration'
GHC_SITE_URL = 'http://host'
GHC_RUNNER_IN_WEBAPP = True
# 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL
GHC_LOG_LEVEL = 30
GHC_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

# Some GetCaps docs are huge. This allows
# caching them for N seconds. Set to -1 to
@@ -1,3 +1,8 @@
import logging

LOGGER = logging.getLogger(__name__)


class Factory:
"""
Object, Function class Factory (Pattern).
@@ -18,9 +23,9 @@ def create_obj(class_string):

# class instance from class object with constructor args
return class_obj()
except Exception, e:
print("cannot create object instance from class '%s' e=%s" %
(class_string, str(e)))
except Exception as e:
LOGGER.error("cannot create object instance from class '%s' e=%s" %
(class_string, str(e)))
raise e

@staticmethod
@@ -41,8 +46,8 @@ def create_class(class_string):
class_obj = getattr(
__import__(module_name, globals(), locals(),
[class_name], -1), class_name)
except Exception, e:
print("cannot create class '%s'" % class_string)
except Exception as e:
LOGGER.error("cannot create class '%s'" % class_string)
raise e

return class_obj
@@ -61,8 +66,8 @@ def create_module(module_string):
try:
module_obj = __import__(module_string, globals(),
locals(), fromlist=[''])
except Exception, e:
print("cannot create module from '%s'" % module_string)
except Exception as e:
LOGGER.error("cannot create module from '%s'" % module_string)
raise e

return module_obj

0 comments on commit 1cc4b3e

Please sign in to comment.
You can’t perform that action at this time.