From 688b64f21c4c2f46d319f493425b36252c9e94b6 Mon Sep 17 00:00:00 2001 From: Patrick Creech Date: Thu, 11 Aug 2016 13:57:57 -0400 Subject: [PATCH] Add task models Add django task models closes #2087 --- platform/pulp/platform/fields.py | 78 ++++++++ platform/pulp/platform/models/__init__.py | 2 + platform/pulp/platform/models/task.py | 213 ++++++++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 platform/pulp/platform/fields.py create mode 100644 platform/pulp/platform/models/task.py diff --git a/platform/pulp/platform/fields.py b/platform/pulp/platform/fields.py new file mode 100644 index 0000000000..92bfbaf75c --- /dev/null +++ b/platform/pulp/platform/fields.py @@ -0,0 +1,78 @@ +""" +Custom Django fields provided by pulp platform +""" +import json + +from django.db import models +from django.utils.translation import six + + +class JSONField(models.TextField): + """ + A custom Django field to serialize data into a text field and vice versa + + """ + def from_db_value(self, value, *args, **kwargs): + """ + Converts a value as returned by the database to a Python object + + :param value: The value to convert to Python + :type value: object + + :param args: unused positional arguments + :type args: list + + :param kwargs: unused keyword arguments + :type kwargs: dict + + :return: A Python representation of value + :rtype: object + """ + if isinstance(value, six.string_types): + return self.to_python(value) + return value + + def to_python(self, value): + """ + Converts the value into the correct Python object + + :param value: The value to convert to Python + :type value: object + + :return: A Python representation of value + :rtype: object + """ + return json.loads(value) + + def get_db_prep_value(self, value, *args, **kwargs): + """ + Converts value to a backend-specific value + + :param value: The value to be converted + :type value: object + + :param args: unused positional arguments + :type args: list + + :param kwargs: unused keyword arguments + :type kwargs: dict + + :return: json string representing the object + :rtype: str + """ + if value is None: + return None + return json.dumps(value) + + def value_to_string(self, obj): + """ + Converts obj to a string. Used to serialize the value of the field + + :param obj: The object to be converted + :type obj: object + + :return: Serialized value + :rtype: str + """ + value = self._get_val_from_obj(obj) + return self.get_db_prep_value(value, None) diff --git a/platform/pulp/platform/models/__init__.py b/platform/pulp/platform/models/__init__.py index ccacf0a17a..8692da70b0 100644 --- a/platform/pulp/platform/models/__init__.py +++ b/platform/pulp/platform/models/__init__.py @@ -6,3 +6,5 @@ from .repository import (Repository, RepositoryGroup, Importer, Distributor, # NOQA RepositoryImporter, RepositoryDistributor, GroupDistributor, # NOQA RepositoryContent) # NOQA + +from .task import ReservedResource, Worker, Task, TaskTag, TaskLock, ScheduledCalls # NOQA diff --git a/platform/pulp/platform/models/task.py b/platform/pulp/platform/models/task.py new file mode 100644 index 0000000000..cae760148a --- /dev/null +++ b/platform/pulp/platform/models/task.py @@ -0,0 +1,213 @@ +""" +Django models related to the Tasking system +""" +from django.db import models + +from pulp.platform.models import Model +from pulp.platform.fields import JSONField + + +class ReservedResource(Model): + """ + Resources that have been reserved + + Fields: + + :cvar resource: The name of the resource reserved for the task. + :type resource: models.TextField + + Relations: + + :cvar task: The task associated with this reservation + :type task: models.ForeignKey + + :cvar worker: The worker associated with this reservation + :type worker: models.ForeignKey + """ + resource = models.TextField() + + task = models.OneToOneField("Task") + worker = models.ForeignKey("Worker") + + +class Worker(Model): + """ + Represents a worker + + Fields: + + :cvar name: The name of the worker, in the format "worker_type@hostname" + :type name: models.TextField + + :cvar last_heartbeat: A timestamp of this worker's last heartbeat + :type last_heartbeat: models.DateTimeField + """ + name = models.TextField(db_index=True, unique=True) + last_heartbeat = models.DateTimeField() + + +class TaskLock(Model): + """ + Locking mechanism for services that utilize active/passive fail-over + + Fields: + + :cvar name: The name of the item that has the lock + :type name: models.TextField + + :cvar timestamp: The time the lock was acquired + :type timestamp: models.DateTimeField + + :cvar lock: The name of the lock acquired + :type lock: models.TextField + + """ + CELERY_BEAT = 'CeleryBeat' + RESOURCE_MANAGER = 'ResourceManager' + LOCK_STRINGS = ( + (CELERY_BEAT, 'Celery Beat Lock'), + (RESOURCE_MANAGER, 'Resource Manager Lock') + ) + + name = models.TextField(db_index=True, unique=True) + timestamp = models.DateTimeField(auto_now_add=True) + lock = models.TextField(unique=True, null=False, choices=LOCK_STRINGS) + + +class Task(Model): + """ + Represents a task + + Fields: + + :cvar group: The group this task belongs to + :type group: models.UUIDField + + :cvar state: The state of the task + :type state: models.TextField + + :cvar started_at: The time the task started executing + :type started_at: models.DateTimeField + + :cvar finished_at: The time the task finished executing + :type finished_at: models.DateTimeField + + :cvar error: Collection of errors that might have occurred while task was running + :type error: models.JSONField + + :cvar result: Return value of the task + :type result: models.JSONField + + Relations: + + :cvar parent: Task that spawned this task (if any) + :type parent: models.ForeignKey + + :cvar worker: The worker that this task is in + :type worker: models.ForeignKey + """ + + WAITING = 'waiting' + SKIPPED = 'skipped' + ACCEPTED = 'accepted' + RUNNING = 'running' + SUSPENDED = 'suspended' + COMPLETED = 'completed' + ERRORED = 'errored' + CANCELED = 'canceled' + STATES = ( + (WAITING, 'Waiting'), + (SKIPPED, 'Skipped'), + (ACCEPTED, 'Accepted'), + (RUNNING, 'Running'), + (SUSPENDED, 'Suspended'), + (COMPLETED, 'Completed'), + (ERRORED, 'Errored'), + (CANCELED, 'Canceled') + ) + group = models.UUIDField(null=True) + state = models.TextField(choices=STATES) + + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + + error = JSONField() + result = JSONField() + + parent = models.ForeignKey("Task", null=True, related_name="spawned_tasks") + worker = models.ForeignKey("Worker", null=True) + + +class TaskTag(Model): + """ + Custom tags for a task + + Fields: + + :cvar name: The name of the tag + :type name: models.TextField + + Relations: + + :cvar task: The task this tag is associated with + :type task: models.ForeignKey + """ + name = models.TextField() + + task = models.ForeignKey("Task", related_name="tags", related_query_name="tag") + + +class ScheduledCalls(Model): + """ + Scheduled Call Request + + Fields: + + :cvar task: The task that should be run on a schedule + :type task: models.TextField + + :cvar enabled: Indicates if schedule should be actively run by the scheduler + :type enabled: models.BooleanField + + :cvar resource: Indicates a unique resource that should be used to find this schedule + :type resource: models.TextField + + :cvar iso_schedule: ISO8601 string representing the schedule + :type iso_schedule: models.TextField + + :cvar schedule: Pickled instance of celery.schedules.schedule that should be run. + :type schedule: models.TextField + + :cvar first_run: The first time this schedule was ran + :type first_run: models.DateTimeField + + :cvar last_run: Last time this schedule was ran + :type last_run: models.DateTimeField + + :cvar total_run_count: Number of times this schedle has ran + :type total_run_count: models.IntegerField + + :cvar last_updated: The last time this schedule was saved to the database + :type last_updated: models.DateTimeField + + :cvar args: Arguments that should be passed to the apply_async function + :type args: models.JSONField + + :cvar kwargs: Keyword arguments that should be passed to the apply_async function + :type kwargs: models.JSONField + """ + task = models.TextField() + enabled = models.BooleanField(default=True) + resource = models.TextField(null=True) + + iso_schedule = models.TextField() + schedule = models.TextField(null=True) + + first_run = models.DateTimeField() + last_run = models.DateTimeField(null=True) + total_run_count = models.IntegerField() + + last_updated = models.DateTimeField() + + args = JSONField() + kwargs = JSONField()