Permalink
Browse files

changed insert and updates on activation for runcaseversions be separ…

…ate to remove issue with replication to slaves. also remove duplicate rcvs, if they exist.
  • Loading branch information...
camd committed Mar 15, 2013
1 parent 3458df5 commit 442d21dc1a48fd514ea79ed70363e7630f6af685
Showing with 154 additions and 132 deletions.
  1. +45 −11 moztrap/model/execution/models.py
  2. +0 −75 moztrap/model/mtmodel.py
  3. +109 −46 tests/model/execution/models/test_run.py
@@ -6,7 +6,7 @@
from django.core.exceptions import ValidationError
from django.db import connection, transaction, models
-from django.db.models import Q
+from django.db.models import Q, Count, Max
from model_utils import Choices
@@ -133,6 +133,8 @@ def _lock_case_versions(self):
run_env_ids = self.environments.values_list("id", flat=True)
# make a list of cvs in order by RunSuite, then SuiteCase.
+ # This list is built from the run / suite / env combination and has
+ # no knowledge of any possibly existing runcaseversions yet.
if len(run_env_ids):
cursor = connection.cursor()
sql = """SELECT DISTINCT cv.id as id
@@ -158,12 +160,34 @@ def _lock_case_versions(self):
cursor.execute(sql)
cv_list = [x[0] for x in cursor.fetchall()]
+
+ # @@@ do we need to check for duplicates?
+ # use itertools.unique_everseen
+ #if len(set(cv_list)) != len(cv_list):
+ # cv_list = itertools.unique_everseen(cv_list)
+
else:
cv_list = []
# delete rcvs that we won't be needing anymore
self._delete_runcaseversions(cv_list)
+ # audit for duplicate rcvs for the same cv.id
+ dups = self.runcaseversions.values("caseversion_id").annotate(
+ num_records=Count("caseversion")).filter(num_records__gt=1)
+ if len(dups) > 0:
+ for dup in dups:
+ # get the runcaseversions, and sort descending by the id
+ # of the results. So the first one is the one with the latest
+ # result. We keep that one and delete the rest.
+ rcv_to_save = self.runcaseversions.annotate(
+ latest_result=Max("results__id")).filter(
+ caseversion=dup["caseversion_id"]).order_by(
+ "-latest_result")[0]
+ self.runcaseversions.filter(
+ caseversion=dup["caseversion_id"]).exclude(
+ id=rcv_to_save.id).delete()
+
# remaining rcvs should be ones we want to keep, and we need to inject
# those ids into the insert/update list for bulk_insert. So create
# a dict mapping cv_id: rcv_id. If one exists, its order field will
@@ -178,21 +202,31 @@ def _lock_case_versions(self):
# an update on insert error.
# runcaseversion objects we will use to bulk create
- rcv_proxies = []
+ rcv_to_update = []
+ rcv_proxies_to_create = []
order = 1
for cv in cv_list:
- kwargs = {"run_id": self.id, "caseversion_id": cv, "order": order}
- try:
- kwargs["id"] = existing_rcv_map[cv]
- except KeyError:
- # this caseversion had not been previously included in the run
- pass
- rcv_proxies.append(RunCaseVersion(**kwargs))
+ if cv in existing_rcv_map:
+ # we will just update the order value
+ rcv_to_update.append({"caseversion_id": cv, "order": order})
+ else:
+ # we need to create a new one
+ kwargs = {
+ "run_id": self.id,
+ "caseversion_id": cv,
+ "order": order
+ }
+ rcv_proxies_to_create.append(RunCaseVersion(**kwargs))
order += 1
+ # update existing rcvs
+ for rcv in rcv_to_update:
+ self.runcaseversions.filter(
+ caseversion=rcv["caseversion_id"]).update(order=rcv["order"])
+
# insert these rcvs in bulk
- self._bulk_insert_new_runcaseversions(rcv_proxies)
+ self._bulk_insert_new_runcaseversions(rcv_proxies_to_create)
self._bulk_update_runcaseversion_environments_for_lock()
@@ -207,7 +241,7 @@ def _delete_runcaseversions(self, cv_list):
def _bulk_insert_new_runcaseversions(self, rcv_proxies):
"""Hook to bulk-insert runcaseversions we know we DO need."""
- RunCaseVersion.objects.bulk_insert_or_update(rcv_proxies)
+ self.runcaseversions.bulk_create(rcv_proxies)
def _bulk_update_runcaseversion_environments_for_lock(self):
View
@@ -152,42 +152,6 @@ def get_query_set(self):
return qs
- def bulk_insert_or_update(self, obj_list):
- """ Bulk insert with a list of model objects"""
-
- if len(obj_list):
- create_fields = [
- field.get_attname_column()[1] for field in obj_list[0]._meta.fields
- ]
- update_fields = set(create_fields) - set([
- "id", "created_by_id", "created_on", "deleted_by_id", "deleted_on"
- ])
-
- def getfield(obj, field):
- value = getattr(obj, field)
- if isinstance(value, (str, datetime.datetime)):
- return "'{0}'".format(value)
- elif value is None:
- return "NULL"
- else:
- return value
-
- values = []
- for obj in obj_list:
- values.append("({0})".format(", ".join(
- ["{0}".format(getfield(obj, field)) for field in create_fields]
- )))
-
- db_table = self.model._meta.db_table
-
- bulk_insert_or_update(
- db_table,
- create_fields,
- update_fields,
- values,
- )
-
-
class MTModel(models.Model):
"""
@@ -476,43 +440,4 @@ def set_default_status(sender, **kwargs):
sender._meta.get_field("status").default = sender.DEFAULT_STATUS
-def bulk_insert_or_update(db_table, create_fields, update_fields, values):
- """
- Bulk insert with a list of fields and values list
-
- On duplicate key, update those values based on the update_fields list.
-
- The concept of this handy manager was borrowed
- from a gist by mmohiudd: https://gist.github.com/3903508
-
- Except I made this work with model objects instead of lists of
- specific fields to update and insert. We figure it out based on the
- fields of the model object.
-
- This provides bulk insert and UPDATE ON DUPLICATE KEY.
-
- """
-
- from django.db import connection, transaction
- cursor = connection.cursor()
-
- base_sql = "INSERT INTO {0} (`{1}`) VALUES {2}".format(
- db_table,
- "`, `".join(create_fields),
- ", ".join(values)
- )
-
- on_duplicates = []
- for field in update_fields:
- on_duplicates.append("`{0}`=VALUES(`{0}`)".format(field, field))
-
- sql = "{0} ON DUPLICATE KEY UPDATE {1}".format(
- base_sql,
- ", ".join(on_duplicates),
- )
-
- cursor.execute(sql)
- transaction.commit_unless_managed()
-
-
class_prepared.connect(set_default_status)
Oops, something went wrong.

0 comments on commit 442d21d

Please sign in to comment.