Skip to content
Browse files

allow stat import and use a uuid to ignore dupes (bug 825347)

  • Loading branch information...
1 parent 1e42af9 commit aa3945f011c3a6190373763786676b5387a05685 @andymckay andymckay committed
Showing with 90 additions and 2 deletions.
  1. +4 −2 metrics/fixtures/metrics.json
  2. +72 −0 metrics/management/commands/migrate.py
  3. +13 −0 metrics/models.py
  4. +1 −0 migrations/02-add-uuid.sql
View
6 metrics/fixtures/metrics.json
@@ -6,7 +6,8 @@
"date": "2009-07-15",
"value": 5,
"name": "addons.downloads.count",
- "key": ""
+ "key": "",
+ "uuid": "1"
}
},
{
@@ -16,7 +17,8 @@
"date": "2009-07-15",
"value": 1305,
"name": "addons.downloads.count.addon",
- "key": "722"
+ "key": "722",
+ "uuid": "2"
}
}
]
View
72 metrics/management/commands/migrate.py
@@ -0,0 +1,72 @@
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+from django.db import utils
+
+from metrics.models import Metric
+
+import json
+
+
+def fix_name(name):
+ # This will do for now, but we should probably do some work here fixing
+ # up the names.
+ name.replace('_', '.')
+ return name.lower()
+
+
+class Command(BaseCommand):
+
+ option_list = BaseCommand.option_list + (
+ make_option('--zamboni', action='store', dest='zamboni',
+ help='Path to the JSON file containing zamboni DB access '
+ 'to the database. This can be a copy of '
+ 'dictionary from settings.'),
+ # For example:
+ # {"ENGINE": "django.db.backends.mysql", "NAME": "zamboni", ...}
+ make_option('--raise', action='store_true', dest='raise',
+ help='Re-raise errors when the occur, rather than skip.')
+ )
+
+ def handle(self, *args, **options):
+ zamboni = options['zamboni']
+ if not zamboni:
+ print 'No path to zamboni given.'
+ return
+
+ db_settings = json.load(open(zamboni, 'r'))
+ backend = utils.load_backend(db_settings['ENGINE'])
+ wrapper = backend.DatabaseWrapper(db_settings)
+ cursor = wrapper.cursor()
+ start = 0
+ limit = 1000 # The number of records to do at a time.
+ count = {'imported': 0, 'skipped': 0, 'errors': 0}
+ sql = 'SELECT * FROM global_stats ORDER BY id LIMIT %s,%s'
+ cursor.execute(sql, (start, limit))
+ results = cursor.fetchall()
+ while results:
+ for row in results:
+ uuid = 'global_stats:%s' % row[0]
+ if not Metric.objects.filter(uuid=uuid).exists():
+ try:
+ Metric.objects.create(uuid=uuid, date=row[3],
+ name=fix_name(row[1]),
+ value=row[2])
+ count['imported'] += 1
+ except:
+ count['errors'] += 1
+ if options['raise']:
+ raise
+
+ else:
+ count['skipped'] += 1
+
+ # Bump our start and re-fetch.
+ start += limit
+ cursor.execute(sql, (start, limit))
+ results = cursor.fetchall()
+
+ print 'Import completed.'
+ print 'Processed:', sum(count.values())
+ for k, v in count.items():
+ print '%s:' % k.title(), v
View
13 metrics/models.py
@@ -1,3 +1,5 @@
+import uuid
+
from django.db import models
from elasticutils import get_es
@@ -5,9 +7,15 @@
class Metric(models.Model):
date = models.DateField(db_index=True)
key = models.CharField(max_length=255, db_index=True)
+ # Let's use the graphite standard, all names are lower case,
+ # delimited by . and in order of specificity.
name = models.CharField(max_length=255, db_index=True)
value = models.IntegerField(db_index=True)
+ # A uuid field so you can spot if this metric has been imported already
+ # into monolith. If not supplied, we'll make one up for you.
+ uuid = models.CharField(max_length=255, db_index=True, unique=True)
+
class Meta:
db_table = 'metric'
@@ -21,6 +29,11 @@ def index(self):
}
conn.index(data, 'monolith', 'metrics/%s/' % self.pk)
+ def save(self, *args, **kw):
+ if not self.uuid:
+ self.uuid = str(uuid.uuid4())
+ super(Metric, self).save(*args, **kw)
+
def unindex(self):
conn = get_es()
conn.delete('monolith', 'metrics', self.pk)
View
1 migrations/02-add-uuid.sql
@@ -0,0 +1 @@
+ALTER TABLE metric ADD COLUMN `uuid` varchar(255) NOT NULL UNIQUE;

0 comments on commit aa3945f

Please sign in to comment.
Something went wrong with that request. Please try again.