Merge d977983 into 1b790a4

LICENSE.3rdparty

@@ -31,3 +31,60 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+================================================================================
+
+The PyFarm distribution contains files from the d3js project. These files are
+distributed under the following conditions:
+
+##d3.js License
+
+Copyright (c) 2012, Michael Bostock
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* The name Michael Bostock may not be used to endorse or promote products
+  derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================================
+
+The PyFarm distribution contains files from the nvd3.js project. These files are
+distributed under the following conditions:
+
+##nvd3.js License
+
+Copyright (c) 2011-2014 [Novus Partners, Inc.][novus]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+[novus]: https://www.novus.com/

pyfarm/master/api/jobs.py

@@ -42,6 +42,7 @@
 from pyfarm.core.enums import STRING_TYPES, NUMERIC_TYPES, WorkState, _WorkState
 from pyfarm.scheduler.tasks import (
     assign_tasks_to_agent, assign_tasks, delete_job)
+from pyfarm.models.statistics.task_event_count import TaskEventCount
 from pyfarm.models.jobtype import JobType, JobTypeVersion
 from pyfarm.models.task import Task
 from pyfarm.models.user import User
@@ -464,31 +465,14 @@ def post(self):
         if not isinstance(by, RANGE_TYPES):
             return (jsonify(error="`by` needs to be of type decimal or int"),
                     BAD_REQUEST)
-        job.by = by
 
         num_tiles = g.json.get("num_tiles", None)
         if not jobtype_version.supports_tiling and num_tiles is not None:
             return (jsonify(error="`num_tiles` is set, but this "
                                   "jobtype does not support tiling."),
                     BAD_REQUEST)
 
-        current_frame = start
-        while current_frame <= end:
-            if num_tiles:
-                for tile in range_(num_tiles - 1):
-                    task = Task()
-                    task.job = job
-                    task.frame = current_frame
-                    task.tile = tile
-                    task.priority = job.priority
-                    db.session.add(task)
-            else:
-                task = Task()
-                task.job = job
-                task.frame = current_frame
-                task.priority = job.priority
-                db.session.add(task)
-            current_frame += by
+        job.alter_frame_range(start, end, by)
 
         db.session.add(job)
         db.session.add_all(software_requirements)
@@ -501,7 +485,7 @@ def post(self):
                                                      "notified_users",
                                                      "tag_requirements"])
         job_data["start"] = start
-        job_data["end"] = min(current_frame, end)
+        job_data["end"] = end
         del job_data["jobtype_version_id"]
         job_data["jobtype"] = job.jobtype_version.jobtype.name
         job_data["jobtype_version"] = job.jobtype_version.version
@@ -848,39 +832,11 @@ def post(self, job_name):
             end = Decimal(json.pop("end", old_last_task.frame))
             by = Decimal(json.pop("by", job.by))
 
-            if end < start:
-                return jsonify(error="`end` must be greater than or equal to "
-                                     "`start`"), BAD_REQUEST
-
-            required_frames = []
-            current_frame = start
-            while current_frame <= end:
-                required_frames.append(current_frame)
-                current_frame += by
-
-            existing_tasks = Task.query.filter_by(job=job).all()
-            frames_to_create = required_frames
-            for task in existing_tasks:
-                if task.frame not in required_frames:
-                    db.session.delete(task)
-                else:
-                    frames_to_create.remove(task.frame)
-
-            for frame in frames_to_create:
-                if job.num_tiles:
-                    for tile in range_(job.num_tiles - 1):
-                        task = Task()
-                        task.job = job
-                        task.frame = current_frame
-                        task.tile = tile
-                        task.priority = job.priority
-                        db.session.add(task)
-                else:
-                    task = Task()
-                    task.job = job
-                    task.frame = current_frame
-                    task.priority = job.priority
-                    db.session.add(task)
+            try:
+                job.alter_frame_range(start, end, by)
+            except ValueError as e:
+                return (jsonify(
+                        error=str(e)), BAD_REQUEST)
 
         if "parents" in g.json:
             parents = []
@@ -1310,6 +1266,22 @@ def post(self, job_name, task_id):
             if task.job.state != old_state and task.job.state == WorkState.DONE:
                 assign_tasks.delay()
 
+        if config.get("enable_statistics") and task.job and state_transition:
+            task_event_count = TaskEventCount(
+                job_queue_id=task.job.job_queue_id)
+            if new_state == "queued":
+                task_event_count.num_restarted = 1
+            elif new_state == "running":
+                task_event_count.num_started = 1
+            elif new_state == "done":
+                task_event_count.num_done = 1
+            elif new_state == "failed":
+                task_event_count.num_failed = 1
+            task_event_count.time_start = datetime.utcnow()
+            task_event_count.time_end = datetime.utcnow()
+            db.session.add(task_event_count)
+            db.session.commit()
+
         return jsonify(task_data), OK
 
     def get(self, job_name, task_id):

pyfarm/master/application.py

@@ -125,6 +125,10 @@ def get_application(**configuration_keywords):
         "TIMESTAMP_FORMAT": config.get("timestamp_format")
     }
 
+    if config.get("enable_statistics"):
+        app_config["SQLALCHEMY_BINDS"] = {
+            "statistics": config.get("statistics_database")}
+
     static_folder = configuration_keywords.pop("static_folder", None)
     if static_folder is None:  # static folder not provided
         import pyfarm.master

pyfarm/master/entrypoints.py

@@ -60,6 +60,9 @@
 from pyfarm.models.tasklog import TaskLog
 from pyfarm.models.gpu import GPU
 from pyfarm.models.jobgroup import JobGroup
+from pyfarm.models.statistics.agent_count import AgentCount
+from pyfarm.models.statistics.task_event_count import TaskEventCount
+from pyfarm.models.statistics.task_count import TaskCount
 from pyfarm.master.utility import timedelta_format
 
 logger = getLogger("master.entrypoints")
@@ -159,6 +162,11 @@ def load_user_interface(app_instance):
         update_version_default_status)
     from pyfarm.master.user_interface.software_version import software_version
     from pyfarm.master.user_interface.jobgroups import jobgroups
+    from pyfarm.master.user_interface.statistics.index import statistics_index
+    from pyfarm.master.user_interface.statistics.agent_counts import (
+        agent_counts)
+    from pyfarm.master.user_interface.statistics.task_events import (
+        task_events)
 
     farm_name = config.get("farm_name")
     app_instance.jinja_env.globals.update({"farm_name": farm_name})
@@ -358,6 +366,17 @@ def load_user_interface(app_instance):
     app_instance.add_url_rule("/jobgroups/",
                               "jobgroups_index_ui", jobgroups, methods=("GET", ))
 
+    app_instance.add_url_rule("/statistics/",
+                              "statistics_index_ui", statistics_index,
+                              methods=("GET", ))
+
+    app_instance.add_url_rule("/statistics/agent_counts",
+                              "agent_counts_ui", agent_counts,
+                              methods=("GET", ))
+    app_instance.add_url_rule("/statistics/task_events",
+                              "task_events_ui", task_events,
+                              methods=("GET", ))
+
 def load_api(app_instance, api_instance):
     """configures flask to serve the api endpoints"""
     from pyfarm.master.api.agents import (

pyfarm/master/etc/master.yml

@@ -35,6 +35,10 @@ debug_queue: false
 database: "sqlite:///pyfarm.sqlite"
 
 
+# Where to store runtime statistics. Same format as "database"
+statistics_database: "sqlite:///pyfarm-statistics.sqlite"
+
+
 # The broker that PyFarm's scheduler should use.  For debugging and
 # development running Redis is the simplest.  For large deployments, or
 # to understand the format of this variable, see:
@@ -80,7 +84,6 @@ api_prefix: /api/v1
 agent_api_url_template: http://{host}:{port}/api/v1
 
 
-
 # Enables or disable the login functionality.  This can be used when
 # debugging or doing development but should not be changed for
 # production.
@@ -212,4 +215,3 @@ job_type_batch_contiguous: true
 ##
 ## END Job Type defaults
 ##
-