Skip to content

#155450742 Add scheduler collection #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Ashaba merged 9 commits into from
May 2, 2018
Merged

#155450742 Add scheduler collection #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
48617de
feat(schedule): schedule collection checks
Ashaba Apr 5, 2018
6d7ef45
feat(schedule): schedule collection checks
Ashaba Apr 5, 2018
8b0fb9c
feat (scheduler): add the scheduler slider
muhallan Apr 10, 2018
522287f
schedule save
Ashaba Apr 11, 2018
0a9cebf
feat(scheduler): implement the editing of the interval of a collection
muhallan Apr 11, 2018
6f01d99
add scheduler
Ashaba Apr 13, 2018
f804286
feat(tasks): background checks with threads
Ashaba Apr 25, 2018
0b01463
feat(celery): implementation of background schedule with celery
Ashaba Apr 26, 2018
6f74c9d
feat(celery): background checks implementation
Ashaba May 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.sample
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
SQLALCHEMY_DATABASE_URI=postgresql://username:password@host:port/name_of_db
FLASK_APP=main.py
REDIS_URL="redis://localhost:6379"
CELERY_BROKER_URL="redis://localhost:6379"
5 changes: 5 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ __pycache__/

.pytest_cache

celery-worker.db
celerybeat-schedule.db
celerybeat.pid
dump.rdb

# Distribution / packaging
.Python
build/
Expand Down
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@

```$ python manage.py db upgrade```

# Celery and Background Tasks
- Start the celery beat
```celery beat -A celery_worker.celery --loglevel=info```
- Start the celery worker
```celery worker -A celery_worker.celery --loglevel=info```


# Running the application
`$ flask run`
Expand Down
Binary file modified application/.DS_Store
View file
Open in desktop
Binary file not shown.
7 changes: 5 additions & 2 deletions application/base_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from flask_sqlalchemy import SQLAlchemy
import logging
from sqlalchemy.sql import func
from sqlalchemy.exc import SQLAlchemyError

from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()

logger = logging.getLogger(__name__)


class Base(db.Model):
"""Base models.
Expand Down Expand Up @@ -32,6 +34,7 @@ def save(self):
return True
except SQLAlchemyError as e:
db.session.rollback()
logger.error("database operation error: ", e)
return False

def __repr__(self):
Expand Down
111 changes: 61 additions & 50 deletions application/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import requests
import json
import re
from application.models import Request, Response, ResponseSummary, ResponseAssertion
from application.models import Response, ResponseSummary, ResponseAssertion
from application.models import Request


def valid_url(url):
Expand All @@ -16,65 +17,75 @@ def valid_url(url):
return True
else:
return False


def run_collection_checks(collection_id, run_from):
checks = Request.filter_by(collection_id=collection_id)
request_check = Request.filter_by(collection_id=collection_id)
response_summary = ResponseSummary(
status='success',
failures=0,
run_from=run_from,
collection_id=collection_id
)

for check in checks:
for req in request_check:
headers = {}
for header in check.headers:
for header in req.headers:
headers[header.key] = header.value
response = make_request(req.url, req.method, headers=headers)
response.request_id = req.id
response.response_summary_id = response_summary.id

failures, status = test_assertions(response, req)

response_summary.failures = failures
response_summary.status = status
response_summary.responses.append(response)
response_summary.save()


def make_request(url, method, headers=None):
request = requests.get(url, headers=headers, verify=False)
try:
response_object = request.json()
except Exception:
response_object = request.content.decode("utf-8")

response = Response(
status_code=request.status_code,
response_time=int(request.elapsed.total_seconds()),
headers=json.dumps(dict(request.headers)),
data=response_object,
)

return response

if check.method == "GET":
result = requests.get(check.url, headers=headers, verify=False)
response = Response(
status_code=result.status_code,
response_time=int(result.elapsed.total_seconds()),
headers=json.dumps(dict(result.headers)),
status='success',
failures=0,
request_id=check.id,
response_summary_id=response_summary.id
)
try:
data = json.dumps(result.json())
except json.decoder.JSONDecodeError:
data = str(result.content)
response.data = data

for assertion in check.assertions:
response_assertion = ResponseAssertion(
assertion_type=assertion.assertion_type,
comparison=assertion.comparison,
value=assertion.value,
status='failed',
request_assertion_id=assertion.id,
response_id=response.id)
if assertion.assertion_type == 'Status Code':
if assertion.value == response.status_code:
response_assertion.status = "success"
def test_assertions(response, req):
failures = 0
status = "success"
for assertion in req.assertions:
response_assertion = ResponseAssertion(
assertion_type=assertion.assertion_type,
comparison=assertion.comparison,
value=assertion.value,
request_assertion_id=assertion.id,
response_id=response.id)

if 'less' in assertion.comparison:
if response.response_time <= assertion.value:
response_assertion.status = status
else:
if 'less' in assertion.comparison:
if response.response_time <= assertion.value:
response_assertion.status = "success"
else:
if response.response_time >= assertion.value:
response_assertion.status = "success"
if response_assertion.status == 'failed':
response.failures += 1
response.status = 'failed'
response.response_assertions.append(response_assertion)
failures += 1
status = "failed"

if assertion.assertion_type == 'Status Code':
if assertion.value == response.status_code:
response_assertion.status = status
else:
failures += 1
status = "failed"
response.failures = failures
response.status = status
response.response_assertions.append(response_assertion)
response.save()

if response.status == 'failed':
response_summary.failures += 1
response_summary.status = 'failed'
response_summary.responses.append(response)
return failures, status

response_summary.save()
6 changes: 2 additions & 4 deletions application/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from application.base_model import Base, db
from sqlalchemy.orm import backref

# many to many relationship between users and teams
team_members = db.Table(
Expand Down Expand Up @@ -32,12 +31,11 @@ class Collection(Base):
name = db.Column(db.String(128), nullable=False)
user_id = db.Column(db.Integer, db.ForeignKey('User.id'), nullable=False)
team_id = db.Column(db.Integer, db.ForeignKey('Team.id'))
requests = db.relationship('Request',
backref='collection', cascade='all, delete-orphan')
response_summary = db.relationship(
'ResponseSummary', backref='collection', lazy=True
)
requests = db.relationship('Request', backref='collection', lazy=True)
interval = db.Column(db.Integer)
requests = db.relationship('Request', backref='collection', cascade='all, delete-orphan')

def __str__(self):
return self.name
Expand Down
47 changes: 47 additions & 0 deletions application/monitoring_tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import os
import redis
import logging
from celery.task.base import periodic_task
from datetime import timedelta
import time
from main import celery

from application.models import Collection
from application.helpers import run_collection_checks

logger = logging.getLogger(__name__)


url = os.environ.get("REDIS_URL") or "redis://localhost:6379"
REDIS_CONN = redis.StrictRedis.from_url(url=url)


try:
REDIS_CONN.ping()
logger.info("Redis is connected!")
except redis.ConnectionError:
logger.error("Redis connection error!")


@celery.task(name='tasks.async_monitoring_checks')
def monitoring_checks():
logger.info("Running Monitoring checks")
try:
collections = Collection.fetch_all()
scheduled_collections = [y for y in collections if y.interval]
for collection in scheduled_collections:
time.sleep(collection.interval)
run_collection_checks(collection.id, "Terminal")
except Exception as e:
logger.error(e)


def async_monitoring_checks():
""" Asynchronous task, called from API endpoint. """
monitoring_checks.delay()
return


@periodic_task(run_every=timedelta(seconds=0))
def periodic_monitoring_checks():
return monitoring_checks()
Binary file modified application/static/.DS_Store
View file
Open in desktop
Binary file not shown.
4 changes: 4 additions & 0 deletions application/static/css/nouislider.min.css
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

98 changes: 98 additions & 0 deletions application/static/css/nouislider.pips.css
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@

/* Base;
*
*/
.noUi-pips,
.noUi-pips * {
-moz-box-sizing: border-box;
box-sizing: border-box;
}
.noUi-pips {
position: absolute;
font: 400 12px Arial;
color: #999;
}

/* Values;
*
*/
.noUi-value {
width: 40px;
position: absolute;
text-align: center;
}
.noUi-value-sub {
color: #ccc;
font-size: 10px;
}

/* Markings;
*
*/
.noUi-marker {
position: absolute;
background: #CCC;
}
.noUi-marker-sub {
background: #AAA;
}
.noUi-marker-large {
background: #AAA;
}

/* Horizontal layout;
*
*/
.noUi-pips-horizontal {
padding: 10px 0;
height: 50px;
top: 100%;
left: 0;
width: 100%;
}
.noUi-value-horizontal {
margin-left: -20px;
padding-top: 20px;
}
.noUi-value-horizontal.noUi-value-sub {
padding-top: 15px;
}

.noUi-marker-horizontal.noUi-marker {
margin-left: -1px;
width: 2px;
height: 5px;
}
.noUi-marker-horizontal.noUi-marker-sub {
height: 10px;
}
.noUi-marker-horizontal.noUi-marker-large {
height: 15px;
}

/* Vertical layout;
*
*/
.noUi-pips-vertical {
padding: 0 10px;
height: 100%;
top: 0;
left: 100%;
}
.noUi-value-vertical {
width: 15px;
margin-left: 20px;
margin-top: -5px;
}

.noUi-marker-vertical.noUi-marker {
width: 5px;
height: 2px;
margin-top: -1px;
}
.noUi-marker-vertical.noUi-marker-sub {
width: 10px;
}
.noUi-marker-vertical.noUi-marker-large {
width: 15px;
}
Loading