[changes] Cleanup adhoc stats

Summary:
Remove a bunch of adhoc stats stuff I added to debug a SEV in Nov.

We have much better tracking now using diamond and this code is really hacky.

Test Plan: make test

Reviewers: vishal, kylec

Reviewed By: kylec

Subscribers: wwu, alexallain

Differential Revision: https://tails.corp.dropbox.com/D84105

changes/backends/jenkins/builder.py

@@ -20,7 +20,6 @@
 from changes.config import db
 from changes.constants import Result, Status
 from changes.db.utils import create_or_update, get_or_create
-from changes.experimental.stats import incr, decr
 from changes.jobs.sync_artifact import sync_artifact
 from changes.jobs.sync_job_step import sync_job_step
 from changes.models import (
@@ -78,7 +77,6 @@ def __init__(self, master_urls=None, job_name=None, token=None, auth=None,
         self.http_session = requests.Session()
 
     def _get_raw_response(self, base_url, path, method='GET', params=None, **kwargs):
-        incr('JenkinsMaster:%s' % base_url)
         url = '{}/{}'.format(base_url, path.lstrip('/'))
 
         kwargs.setdefault('allow_redirects', False)
@@ -102,7 +100,6 @@ def _get_raw_response(self, base_url, path, method='GET', params=None, **kwargs)
             self.logger.exception(exception_msg, *attrs)
             raise Exception(exception_msg % attrs)
 
-        decr('JenkinsMaster:%s' % base_url)
         return resp.text
 
     def _get_json_response(self, base_url, path, *args, **kwargs):

changes/backends/jenkins/buildsteps/collector.py

@@ -7,7 +7,6 @@
 from changes.config import db
 from changes.constants import Result, Status
 from changes.db.utils import get_or_create, try_create
-from changes.experimental.stats import incr
 from changes.jobs.sync_job_step import sync_job_step
 from changes.models import FailureReason, JobPhase, JobStep
 
@@ -145,9 +144,6 @@ def _expand_job(self, phase, job_config):
                     break
                 except Exception as ex:
                     logging.exception("Failed to create jobstep")
-                    if master:
-                        master_key = 'JM:%s' % master
-                        incr(master_key)
                     exn = ex
 
             if not success:

changes/jobs/cleanup_tasks.py

@@ -4,7 +4,6 @@
 
 from changes.config import queue
 from changes.constants import Status
-from changes.experimental.stats import RCount, incr
 from changes.models import Task
 from changes.queue.task import TrackedTask, tracked_task
 
@@ -20,24 +19,22 @@ def cleanup_tasks():
 
     Additionally remove any old Task entries which are completed.
     """
-    with RCount('cleanup_tasks'):
-        now = datetime.utcnow()
-
-        pending_tasks = Task.query.filter(
-            Task.status != Status.finished,
-            Task.date_modified < now - CHECK_TIME,
+    now = datetime.utcnow()
+
+    pending_tasks = Task.query.filter(
+        Task.status != Status.finished,
+        Task.date_modified < now - CHECK_TIME,
+    )
+
+    for task in pending_tasks:
+        task_func = TrackedTask(queue.get_task(task.task_name))
+        task_func.delay(
+            task_id=task.task_id.hex,
+            parent_task_id=task.parent_id.hex if task.parent_id else None,
+            **task.data['kwargs']
         )
 
-        for task in pending_tasks:
-            incr('cleanup_unfinished')
-            task_func = TrackedTask(queue.get_task(task.task_name))
-            task_func.delay(
-                task_id=task.task_id.hex,
-                parent_task_id=task.parent_id.hex if task.parent_id else None,
-                **task.data['kwargs']
-            )
-
-        Task.query.filter(
-            Task.status == Status.finished,
-            Task.date_modified < now - EXPIRE_TIME,
-        ).delete()
+    Task.query.filter(
+        Task.status == Status.finished,
+        Task.date_modified < now - EXPIRE_TIME,
+    ).delete()

changes/jobs/create_job.py

@@ -3,7 +3,6 @@
 from changes.backends.base import UnrecoverableException
 from changes.config import db
 from changes.constants import Status, Result
-from changes.experimental.stats import RCount
 from changes.jobs.sync_job import sync_job
 from changes.models import Job, JobPlan, ProjectStatus
 from changes.queue.task import tracked_task
@@ -20,46 +19,45 @@ def abort_create(task):
 
 @tracked_task(on_abort=abort_create, max_retries=10)
 def create_job(job_id):
-    with RCount('create_job'):
-        job = Job.query.get(job_id)
-        if not job:
-            return
-
-        if job.project.status == ProjectStatus.inactive:
-            current_app.logger.warn('Project is not active: %s', job.project.slug)
-            job.status = Status.finished
-            job.result = Result.failed
-            db.session.add(job)
-            db.session.flush()
-            return
-
-        # we might already be marked as finished for various reasons
-        # (such as aborting the task)
-        if job.status == Status.finished:
-            return
-
-        jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)
-        if implementation is None:
-            # TODO(dcramer): record a FailureReason?
-            job.status = Status.finished
-            job.result = Result.failed
-            db.session.add(job)
-            db.session.flush()
-            current_app.logger.exception('No build plan set %s', job_id)
-            return
-
-        try:
-            implementation.execute(job=job)
-        except UnrecoverableException:
-            job.status = Status.finished
-            job.result = Result.aborted
-            db.session.add(job)
-            db.session.flush()
-            current_app.logger.exception('Unrecoverable exception creating %s', job_id)
-            return
-
-        sync_job.delay(
-            job_id=job.id.hex,
-            task_id=job.id.hex,
-            parent_task_id=job.build_id.hex,
-        )
+    job = Job.query.get(job_id)
+    if not job:
+        return
+
+    if job.project.status == ProjectStatus.inactive:
+        current_app.logger.warn('Project is not active: %s', job.project.slug)
+        job.status = Status.finished
+        job.result = Result.failed
+        db.session.add(job)
+        db.session.flush()
+        return
+
+    # we might already be marked as finished for various reasons
+    # (such as aborting the task)
+    if job.status == Status.finished:
+        return
+
+    jobplan, implementation = JobPlan.get_build_step_for_job(job_id=job.id)
+    if implementation is None:
+        # TODO(dcramer): record a FailureReason?
+        job.status = Status.finished
+        job.result = Result.failed
+        db.session.add(job)
+        db.session.flush()
+        current_app.logger.exception('No build plan set %s', job_id)
+        return
+
+    try:
+        implementation.execute(job=job)
+    except UnrecoverableException:
+        job.status = Status.finished
+        job.result = Result.aborted
+        db.session.add(job)
+        db.session.flush()
+        current_app.logger.exception('Unrecoverable exception creating %s', job_id)
+        return
+
+    sync_job.delay(
+        job_id=job.id.hex,
+        task_id=job.id.hex,
+        parent_task_id=job.build_id.hex,
+    )

changes/jobs/import_repo.py

@@ -5,7 +5,6 @@
 from datetime import datetime
 
 from changes.config import db
-from changes.experimental.stats import RCount
 from changes.models import Repository, RepositoryStatus
 from changes.queue.task import tracked_task
 
@@ -14,49 +13,48 @@
 
 @tracked_task(max_retries=None)
 def import_repo(repo_id, parent=None):
-    with RCount('import_repo'):
-        repo = Repository.query.get(repo_id)
-        if not repo:
-            logger.error('Repository %s not found', repo_id)
-            return
-
-        vcs = repo.get_vcs()
-        if vcs is None:
-            logger.warning('Repository %s has no VCS backend set', repo.id)
-            return
-
-        if repo.status == RepositoryStatus.inactive:
-            logger.info('Repository %s is inactive', repo.id)
-            return
-
-        Repository.query.filter(
-            Repository.id == repo.id,
-        ).update({
-            'last_update_attempt': datetime.utcnow(),
-        }, synchronize_session=False)
+    repo = Repository.query.get(repo_id)
+    if not repo:
+        logger.error('Repository %s not found', repo_id)
+        return
+
+    vcs = repo.get_vcs()
+    if vcs is None:
+        logger.warning('Repository %s has no VCS backend set', repo.id)
+        return
+
+    if repo.status == RepositoryStatus.inactive:
+        logger.info('Repository %s is inactive', repo.id)
+        return
+
+    Repository.query.filter(
+        Repository.id == repo.id,
+    ).update({
+        'last_update_attempt': datetime.utcnow(),
+    }, synchronize_session=False)
+    db.session.commit()
+
+    if vcs.exists():
+        vcs.update()
+    else:
+        vcs.clone()
+
+    for commit in vcs.log(parent=parent):
+        revision, created = commit.save(repo)
         db.session.commit()
-
-        if vcs.exists():
-            vcs.update()
-        else:
-            vcs.clone()
-
-        for commit in vcs.log(parent=parent):
-            revision, created = commit.save(repo)
-            db.session.commit()
-            parent = commit.id
-
-        Repository.query.filter(
-            Repository.id == repo.id,
-        ).update({
-            'last_update': datetime.utcnow(),
-            'status': RepositoryStatus.active,
-        }, synchronize_session=False)
-        db.session.commit()
-
-        if parent:
-            import_repo.delay(
-                repo_id=repo.id.hex,
-                task_id=repo.id.hex,
-                parent=parent,
-            )
+        parent = commit.id
+
+    Repository.query.filter(
+        Repository.id == repo.id,
+    ).update({
+        'last_update': datetime.utcnow(),
+        'status': RepositoryStatus.active,
+    }, synchronize_session=False)
+    db.session.commit()
+
+    if parent:
+        import_repo.delay(
+            repo_id=repo.id.hex,
+            task_id=repo.id.hex,
+            parent=parent,
+        )

changes/jobs/signals.py

@@ -1,5 +1,4 @@
 from flask import current_app
-from changes.experimental.stats import RCount
 
 from changes.queue.task import tracked_task
 from changes.utils.imports import import_string
@@ -11,22 +10,20 @@ class SuspiciousOperation(Exception):
 
 @tracked_task
 def fire_signal(signal, kwargs):
-    with RCount('fire_signal'):
-        for listener, l_signal in current_app.config['EVENT_LISTENERS']:
-            if l_signal == signal:
-                run_event_listener.delay(
-                    listener=listener,
-                    signal=signal,
-                    kwargs=kwargs,
-                )
+    for listener, l_signal in current_app.config['EVENT_LISTENERS']:
+        if l_signal == signal:
+            run_event_listener.delay(
+                listener=listener,
+                signal=signal,
+                kwargs=kwargs,
+            )
 
 
 @tracked_task
 def run_event_listener(listener, signal, kwargs):
-    with RCount('run_event_listener'):
-        # simple check to make sure this is registered
-        if not any(l == listener for l, _ in current_app.config['EVENT_LISTENERS']):
-            raise SuspiciousOperation('%s is not a registered event listener' % (listener,))
+    # simple check to make sure this is registered
+    if not any(l == listener for l, _ in current_app.config['EVENT_LISTENERS']):
+        raise SuspiciousOperation('%s is not a registered event listener' % (listener,))
 
-        func = import_string(listener)
-        func(**kwargs)
+    func = import_string(listener)
+    func(**kwargs)