From 2c3c2796410ec8a5dbd61645f946ae513a9da280 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 7 Apr 2016 08:21:13 +0200 Subject: [PATCH 01/17] Initial work on database based system symbols --- ...e_dsymsymbol_bundle_address__add_dsymsd.py | 611 ++++++++++++++++++ src/sentry/models/dsymfile.py | 72 ++- src/sentry/runner/__init__.py | 1 + .../runner/commands/import_system_symbols.py | 124 ++++ 4 files changed, 807 insertions(+), 1 deletion(-) create mode 100644 src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py create mode 100644 src/sentry/runner/commands/import_system_symbols.py diff --git a/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py b/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py new file mode 100644 index 00000000000000..49f6317da8f31c --- /dev/null +++ b/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py @@ -0,0 +1,611 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'DSymSymbol' + db.create_table('sentry_dsymsymbol', ( + ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), + ('bundle', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymBundle'])), + ('address', self.gf('sentry.db.models.fields.bounded.BoundedBigIntegerField')()), + ('symbol', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('sentry', ['DSymSymbol']) + + # Adding unique constraint on 'DSymSymbol', fields ['bundle', 'address'] + db.create_unique('sentry_dsymsymbol', ['bundle_id', 'address']) + + # Adding model 'DSymSDK' + db.create_table('sentry_dsymsdk', ( + ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), + ('dsym_type', self.gf('django.db.models.fields.CharField')(max_length=20, db_index=True)), + ('sdk_name', self.gf('django.db.models.fields.CharField')(max_length=20)), + ('version_major', self.gf('django.db.models.fields.IntegerField')()), + ('version_minor', self.gf('django.db.models.fields.IntegerField')()), + ('version_patchlevel', self.gf('django.db.models.fields.IntegerField')()), + ('version_build', self.gf('django.db.models.fields.CharField')(max_length=40)), + )) + db.send_create_signal('sentry', ['DSymSDK']) + + # Adding index on 'DSymSDK', fields ['version_major', 'version_minor', 'version_patchlevel', 'version_build'] + db.create_index('sentry_dsymsdk', ['version_major', 'version_minor', 'version_patchlevel', 'version_build']) + + # Adding model 'DSymBundle' + db.create_table('sentry_dsymbundle', ( + ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), + ('sdk', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymSDK'])), + ('cpu_name', self.gf('django.db.models.fields.CharField')(max_length=40)), + ('object_path', self.gf('django.db.models.fields.TextField')(db_index=True)), + ('uuid', self.gf('django.db.models.fields.CharField')(max_length=36, db_index=True)), + )) + db.send_create_signal('sentry', ['DSymBundle']) + + + def backwards(self, orm): + # Removing index on 'DSymSDK', fields ['version_major', 'version_minor', 'version_patchlevel', 'version_build'] + db.delete_index('sentry_dsymsdk', ['version_major', 'version_minor', 'version_patchlevel', 'version_build']) + + # Removing unique constraint on 'DSymSymbol', fields ['bundle', 'address'] + db.delete_unique('sentry_dsymsymbol', ['bundle_id', 'address']) + + # Deleting model 'DSymSymbol' + db.delete_table('sentry_dsymsymbol') + + # Deleting model 'DSymSDK' + db.delete_table('sentry_dsymsdk') + + # Deleting model 'DSymBundle' + db.delete_table('sentry_dsymbundle') + + + models = { + 'sentry.activity': { + 'Meta': {'object_name': 'Activity'}, + 'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {'null': 'True'}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']", 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'ident': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'type': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']", 'null': 'True'}) + }, + 'sentry.apikey': { + 'Meta': {'object_name': 'ApiKey'}, + 'allowed_origins': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}), + 'label': ('django.db.models.fields.CharField', [], {'default': "'Default'", 'max_length': '64', 'blank': 'True'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'key_set'", 'to': "orm['sentry.Organization']"}), + 'scopes': ('django.db.models.fields.BigIntegerField', [], {'default': 'None'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0', 'db_index': 'True'}) + }, + 'sentry.auditlogentry': { + 'Meta': {'object_name': 'AuditLogEntry'}, + 'actor': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'blank': 'True', 'related_name': "'audit_actors'", 'null': 'True', 'to': "orm['sentry.User']"}), + 'actor_key': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.ApiKey']", 'null': 'True', 'blank': 'True'}), + 'actor_label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'event': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'ip_address': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']"}), + 'target_object': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'null': 'True'}), + 'target_user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'blank': 'True', 'related_name': "'audit_targets'", 'null': 'True', 'to': "orm['sentry.User']"}) + }, + 'sentry.authidentity': { + 'Meta': {'unique_together': "(('auth_provider', 'ident'), ('auth_provider', 'user'))", 'object_name': 'AuthIdentity'}, + 'auth_provider': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.AuthProvider']"}), + 'data': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'ident': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'last_synced': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_verified': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']"}) + }, + 'sentry.authprovider': { + 'Meta': {'object_name': 'AuthProvider'}, + 'config': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'default_global_access': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'default_role': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '50'}), + 'default_teams': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['sentry.Team']", 'symmetrical': 'False', 'blank': 'True'}), + 'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']", 'unique': 'True'}), + 'provider': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'sync_time': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'null': 'True'}) + }, + 'sentry.broadcast': { + 'Meta': {'object_name': 'Broadcast'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'date_expires': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2016, 4, 13, 0, 0)', 'null': 'True', 'blank': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), + 'link': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'message': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'upstream_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}) + }, + 'sentry.broadcastseen': { + 'Meta': {'unique_together': "(('broadcast', 'user'),)", 'object_name': 'BroadcastSeen'}, + 'broadcast': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Broadcast']"}), + 'date_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']"}) + }, + 'sentry.counter': { + 'Meta': {'object_name': 'Counter', 'db_table': "'sentry_projectcounter'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'unique': 'True'}), + 'value': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}) + }, + 'sentry.dsymbundle': { + 'Meta': {'object_name': 'DSymBundle'}, + 'cpu_name': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'object_path': ('django.db.models.fields.TextField', [], {'db_index': 'True'}), + 'sdk': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymSDK']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '36', 'db_index': 'True'}) + }, + 'sentry.dsymsdk': { + 'Meta': {'object_name': 'DSymSDK', 'index_together': "[('version_major', 'version_minor', 'version_patchlevel', 'version_build')]"}, + 'dsym_type': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'sdk_name': ('django.db.models.fields.CharField', [], {'max_length': '20'}), + 'version_build': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'version_major': ('django.db.models.fields.IntegerField', [], {}), + 'version_minor': ('django.db.models.fields.IntegerField', [], {}), + 'version_patchlevel': ('django.db.models.fields.IntegerField', [], {}) + }, + 'sentry.dsymsymbol': { + 'Meta': {'unique_together': "[('bundle', 'address')]", 'object_name': 'DSymSymbol'}, + 'address': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), + 'bundle': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymBundle']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'symbol': ('django.db.models.fields.TextField', [], {}) + }, + 'sentry.event': { + 'Meta': {'unique_together': "(('project_id', 'event_id'),)", 'object_name': 'Event', 'db_table': "'sentry_message'", 'index_together': "(('group_id', 'datetime'),)"}, + 'data': ('sentry.db.models.fields.node.NodeField', [], {'null': 'True', 'blank': 'True'}), + 'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'event_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'db_column': "'message_id'"}), + 'group_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'platform': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'time_spent': ('sentry.db.models.fields.bounded.BoundedIntegerField', [], {'null': 'True'}) + }, + 'sentry.eventmapping': { + 'Meta': {'unique_together': "(('project_id', 'event_id'),)", 'object_name': 'EventMapping'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'event_id': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'group_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}) + }, + 'sentry.eventtag': { + 'Meta': {'unique_together': "(('event_id', 'key_id', 'value_id'),)", 'object_name': 'EventTag'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'event_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), + 'project_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), + 'value_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}) + }, + 'sentry.eventuser': { + 'Meta': {'unique_together': "(('project', 'ident'), ('project', 'hash'))", 'object_name': 'EventUser', 'index_together': "(('project', 'email'), ('project', 'username'), ('project', 'ip_address'))"}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True'}), + 'hash': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'ident': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'ip_address': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'username': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}) + }, + 'sentry.file': { + 'Meta': {'object_name': 'File'}, + 'blob': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'legacy_blob'", 'null': 'True', 'to': "orm['sentry.FileBlob']"}), + 'blobs': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['sentry.FileBlob']", 'through': "orm['sentry.FileBlobIndex']", 'symmetrical': 'False'}), + 'checksum': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True'}), + 'headers': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'path': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'size': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'null': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '64'}) + }, + 'sentry.fileblob': { + 'Meta': {'object_name': 'FileBlob'}, + 'checksum': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'path': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'size': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'null': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}) + }, + 'sentry.fileblobindex': { + 'Meta': {'unique_together': "(('file', 'blob', 'offset'),)", 'object_name': 'FileBlobIndex'}, + 'blob': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.FileBlob']"}), + 'file': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.File']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'offset': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {}) + }, + 'sentry.globaldsymfile': { + 'Meta': {'object_name': 'GlobalDSymFile'}, + 'cpu_name': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'file': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.File']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'object_name': ('django.db.models.fields.TextField', [], {}), + 'uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}) + }, + 'sentry.group': { + 'Meta': {'unique_together': "(('project', 'short_id'),)", 'object_name': 'Group', 'db_table': "'sentry_groupedmessage'", 'index_together': "(('project', 'first_release'),)"}, + 'active_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), + 'culprit': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_column': "'view'", 'blank': 'True'}), + 'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {'null': 'True', 'blank': 'True'}), + 'first_release': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Release']", 'null': 'True', 'on_delete': 'models.PROTECT'}), + 'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'is_public': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'level': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '40', 'db_index': 'True', 'blank': 'True'}), + 'logger': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '64', 'db_index': 'True', 'blank': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'num_comments': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0', 'null': 'True'}), + 'platform': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}), + 'resolved_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}), + 'score': ('sentry.db.models.fields.bounded.BoundedIntegerField', [], {'default': '0'}), + 'short_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0', 'db_index': 'True'}), + 'time_spent_count': ('sentry.db.models.fields.bounded.BoundedIntegerField', [], {'default': '0'}), + 'time_spent_total': ('sentry.db.models.fields.bounded.BoundedIntegerField', [], {'default': '0'}), + 'times_seen': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '1', 'db_index': 'True'}) + }, + 'sentry.groupassignee': { + 'Meta': {'object_name': 'GroupAssignee', 'db_table': "'sentry_groupasignee'"}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'assignee_set'", 'unique': 'True', 'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'assignee_set'", 'to': "orm['sentry.Project']"}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'sentry_assignee_set'", 'to': "orm['sentry.User']"}) + }, + 'sentry.groupbookmark': { + 'Meta': {'unique_together': "(('project', 'user', 'group'),)", 'object_name': 'GroupBookmark'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'bookmark_set'", 'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'bookmark_set'", 'to': "orm['sentry.Project']"}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'sentry_bookmark_set'", 'to': "orm['sentry.User']"}) + }, + 'sentry.groupemailthread': { + 'Meta': {'unique_together': "(('email', 'group'), ('email', 'msgid'))", 'object_name': 'GroupEmailThread'}, + 'date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'groupemail_set'", 'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'msgid': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'groupemail_set'", 'to': "orm['sentry.Project']"}) + }, + 'sentry.grouphash': { + 'Meta': {'unique_together': "(('project', 'hash'),)", 'object_name': 'GroupHash'}, + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']", 'null': 'True'}), + 'hash': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}) + }, + 'sentry.groupmeta': { + 'Meta': {'unique_together': "(('group', 'key'),)", 'object_name': 'GroupMeta'}, + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'sentry.groupredirect': { + 'Meta': {'object_name': 'GroupRedirect'}, + 'group_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'db_index': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'previous_group_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'unique': 'True'}) + }, + 'sentry.groupresolution': { + 'Meta': {'object_name': 'GroupResolution'}, + 'datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']", 'unique': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'release': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Release']"}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}) + }, + 'sentry.grouprulestatus': { + 'Meta': {'unique_together': "(('rule', 'group'),)", 'object_name': 'GroupRuleStatus'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'last_active': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'rule': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Rule']"}), + 'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}) + }, + 'sentry.groupseen': { + 'Meta': {'unique_together': "(('user', 'group'),)", 'object_name': 'GroupSeen'}, + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']", 'db_index': 'False'}) + }, + 'sentry.groupsnooze': { + 'Meta': {'object_name': 'GroupSnooze'}, + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']", 'unique': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'until': ('django.db.models.fields.DateTimeField', [], {}) + }, + 'sentry.grouptagkey': { + 'Meta': {'unique_together': "(('project', 'group', 'key'),)", 'object_name': 'GroupTagKey'}, + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}), + 'values_seen': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}) + }, + 'sentry.grouptagvalue': { + 'Meta': {'unique_together': "(('project', 'key', 'value', 'group'),)", 'object_name': 'GroupTagValue', 'db_table': "'sentry_messagefiltervalue'"}, + 'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'grouptag'", 'to': "orm['sentry.Group']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'grouptag'", 'null': 'True', 'to': "orm['sentry.Project']"}), + 'times_seen': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + 'sentry.helppage': { + 'Meta': {'object_name': 'HelpPage'}, + 'content': ('django.db.models.fields.TextField', [], {}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'is_visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '64', 'unique': 'True', 'null': 'True'}), + 'priority': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '50'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}) + }, + 'sentry.lostpasswordhash': { + 'Meta': {'object_name': 'LostPasswordHash'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'hash': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']", 'unique': 'True'}) + }, + 'sentry.option': { + 'Meta': {'object_name': 'Option'}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}), + 'last_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'value': ('sentry.db.models.fields.pickle.UnicodePickledObjectField', [], {}) + }, + 'sentry.organization': { + 'Meta': {'object_name': 'Organization'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'default_role': ('django.db.models.fields.CharField', [], {'default': "'member'", 'max_length': '32'}), + 'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '1'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'members': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'org_memberships'", 'symmetrical': 'False', 'through': "orm['sentry.OrganizationMember']", 'to': "orm['sentry.User']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}) + }, + 'sentry.organizationaccessrequest': { + 'Meta': {'unique_together': "(('team', 'member'),)", 'object_name': 'OrganizationAccessRequest'}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'member': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.OrganizationMember']"}), + 'team': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Team']"}) + }, + 'sentry.organizationmember': { + 'Meta': {'unique_together': "(('organization', 'user'), ('organization', 'email'))", 'object_name': 'OrganizationMember'}, + 'counter': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'has_global_access': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'member_set'", 'to': "orm['sentry.Organization']"}), + 'role': ('django.db.models.fields.CharField', [], {'default': "'member'", 'max_length': '32'}), + 'teams': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['sentry.Team']", 'symmetrical': 'False', 'through': "orm['sentry.OrganizationMemberTeam']", 'blank': 'True'}), + 'type': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '50', 'blank': 'True'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'blank': 'True', 'related_name': "'sentry_orgmember_set'", 'null': 'True', 'to': "orm['sentry.User']"}) + }, + 'sentry.organizationmemberteam': { + 'Meta': {'unique_together': "(('team', 'organizationmember'),)", 'object_name': 'OrganizationMemberTeam', 'db_table': "'sentry_organizationmember_teams'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedAutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'organizationmember': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.OrganizationMember']"}), + 'team': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Team']"}) + }, + 'sentry.organizationonboardingtask': { + 'Meta': {'unique_together': "(('organization', 'task'),)", 'object_name': 'OrganizationOnboardingTask'}, + 'data': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'date_completed': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']"}), + 'project_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {}), + 'task': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']", 'null': 'True'}) + }, + 'sentry.organizationoption': { + 'Meta': {'unique_together': "(('organization', 'key'),)", 'object_name': 'OrganizationOption', 'db_table': "'sentry_organizationoptions'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']"}), + 'value': ('sentry.db.models.fields.pickle.UnicodePickledObjectField', [], {}) + }, + 'sentry.project': { + 'Meta': {'unique_together': "(('team', 'slug'), ('organization', 'slug'))", 'object_name': 'Project'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'first_event': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'forced_color': ('django.db.models.fields.CharField', [], {'max_length': '6', 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']"}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'null': 'True'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0', 'db_index': 'True'}), + 'team': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Team']"}) + }, + 'sentry.projectbookmark': { + 'Meta': {'unique_together': "(('project_id', 'user'),)", 'object_name': 'ProjectBookmark'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project_id': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']"}) + }, + 'sentry.projectdsymfile': { + 'Meta': {'unique_together': "(('project', 'uuid'),)", 'object_name': 'ProjectDSymFile'}, + 'cpu_name': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'file': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.File']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'object_name': ('django.db.models.fields.TextField', [], {}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '36'}) + }, + 'sentry.projectkey': { + 'Meta': {'object_name': 'ProjectKey'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'related_name': "'key_set'", 'to': "orm['sentry.Project']"}), + 'public_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'unique': 'True', 'null': 'True'}), + 'roles': ('django.db.models.fields.BigIntegerField', [], {'default': '1'}), + 'secret_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'unique': 'True', 'null': 'True'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0', 'db_index': 'True'}) + }, + 'sentry.projectoption': { + 'Meta': {'unique_together': "(('project', 'key'),)", 'object_name': 'ProjectOption', 'db_table': "'sentry_projectoptions'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'value': ('sentry.db.models.fields.pickle.UnicodePickledObjectField', [], {}) + }, + 'sentry.release': { + 'Meta': {'unique_together': "(('project', 'version'),)", 'object_name': 'Release'}, + 'data': ('jsonfield.fields.JSONField', [], {'default': '{}'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'date_released': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'date_started': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'new_groups': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}), + 'owner': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']", 'null': 'True', 'blank': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'ref': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '64'}) + }, + 'sentry.releasefile': { + 'Meta': {'unique_together': "(('release', 'ident'),)", 'object_name': 'ReleaseFile'}, + 'file': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.File']"}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'ident': ('django.db.models.fields.CharField', [], {'max_length': '40'}), + 'name': ('django.db.models.fields.TextField', [], {}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'release': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Release']"}) + }, + 'sentry.rule': { + 'Meta': {'object_name': 'Rule'}, + 'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}) + }, + 'sentry.savedsearch': { + 'Meta': {'unique_together': "(('project', 'name'),)", 'object_name': 'SavedSearch'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'is_default': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'query': ('django.db.models.fields.TextField', [], {}) + }, + 'sentry.savedsearchuserdefault': { + 'Meta': {'unique_together': "(('project', 'user'),)", 'object_name': 'SavedSearchUserDefault', 'db_table': "'sentry_savedsearch_userdefault'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'savedsearch': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.SavedSearch']"}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']"}) + }, + 'sentry.tagkey': { + 'Meta': {'unique_together': "(('project', 'key'),)", 'object_name': 'TagKey', 'db_table': "'sentry_filterkey'"}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'label': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}), + 'values_seen': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}) + }, + 'sentry.tagvalue': { + 'Meta': {'unique_together': "(('project', 'key', 'value'),)", 'object_name': 'TagValue', 'db_table': "'sentry_filtervalue'"}, + 'data': ('sentry.db.models.fields.gzippeddict.GzippedDictField', [], {'null': 'True', 'blank': 'True'}), + 'first_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'db_index': 'True'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}), + 'times_seen': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + 'sentry.team': { + 'Meta': {'unique_together': "(('organization', 'slug'),)", 'object_name': 'Team'}, + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'organization': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Organization']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), + 'status': ('sentry.db.models.fields.bounded.BoundedPositiveIntegerField', [], {'default': '0'}) + }, + 'sentry.user': { + 'Meta': {'object_name': 'User', 'db_table': "'auth_user'"}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedAutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_managed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_column': "'first_name'", 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}) + }, + 'sentry.useroption': { + 'Meta': {'unique_together': "(('user', 'project', 'key'),)", 'object_name': 'UserOption'}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']", 'null': 'True'}), + 'user': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.User']"}), + 'value': ('sentry.db.models.fields.pickle.UnicodePickledObjectField', [], {}) + }, + 'sentry.userreport': { + 'Meta': {'object_name': 'UserReport', 'index_together': "(('project', 'event_id'), ('project', 'date_added'))"}, + 'comments': ('django.db.models.fields.TextField', [], {}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'event_id': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'group': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Group']", 'null': 'True'}), + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'project': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.Project']"}) + } + } + + complete_apps = ['sentry'] \ No newline at end of file diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index 32efb607b5bfba..1419ca740fe0fc 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -20,7 +20,8 @@ except ImportError: have_symsynd = False -from sentry.db.models import FlexibleForeignKey, Model, sane_repr +from sentry.db.models import FlexibleForeignKey, Model, BoundedBigIntegerField, \ + sane_repr from sentry.models.file import File from sentry.utils.zip import safe_extract_zip @@ -30,6 +31,75 @@ } +''' +system symbols: + + architecture VARCHAR(20) + sdk-version MAJOR.MINOR.PATCHLEVEL-BUILD + uuid UUID + path VARCHAR(255) + address BIGINT + symbol TEXT + +lookup logic: + + primary lookup: + uuid -> exact match + address -> lower than or equal to reference address + limit 1 + + secondary lookup: + path -> exact match + sdk-version -> fuzzy match + address -> lower than or equal to reference address + limit 1 +''' + + +class DSymSDK(Model): + __core__ = False + dsym_type = models.CharField(max_length=20, db_index=True) + sdk_name = models.CharField(max_length=20) + version_major = models.IntegerField() + version_minor = models.IntegerField() + version_patchlevel = models.IntegerField() + version_build = models.CharField(max_length=40) + + class Meta: + app_label = 'sentry' + db_table = 'sentry_dsymsdk' + index_together = [ + ('version_major', 'version_minor', 'version_patchlevel', + 'version_build'), + ] + + +class DSymBundle(Model): + __core__ = False + sdk = FlexibleForeignKey('sentry.DSymSDK') + cpu_name = models.CharField(max_length=40) + object_path = models.TextField(db_index=True) + uuid = models.CharField(max_length=36, db_index=True) + + class Meta: + app_label = 'sentry' + db_table = 'sentry_dsymbundle' + + +class DSymSymbol(Model): + __core__ = False + bundle = FlexibleForeignKey('sentry.DSymBundle') + address = BoundedBigIntegerField(db_index=True) + symbol = models.TextField() + + class Meta: + app_label = 'sentry' + db_table = 'sentry_dsymsymbol' + unique_together = [ + ('bundle', 'address'), + ] + + class CommonDSymFile(Model): """ A single dsym file that is associated with a project. diff --git a/src/sentry/runner/__init__.py b/src/sentry/runner/__init__.py index 64ead4957e141c..97d2f46e820c36 100755 --- a/src/sentry/runner/__init__.py +++ b/src/sentry/runner/__init__.py @@ -61,6 +61,7 @@ def cli(ctx, config): 'sentry.runner.commands.start.start', 'sentry.runner.commands.tsdb.tsdb', 'sentry.runner.commands.upgrade.upgrade', + 'sentry.runner.commands.import_system_symbols.import_system_symbols', )) diff --git a/src/sentry/runner/commands/import_system_symbols.py b/src/sentry/runner/commands/import_system_symbols.py new file mode 100644 index 00000000000000..9e73c882f6a94a --- /dev/null +++ b/src/sentry/runner/commands/import_system_symbols.py @@ -0,0 +1,124 @@ +""" +sentry.runner.commands.import_system_symbols +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details. +:license: BSD, see LICENSE for more details. +""" +from __future__ import absolute_import + +import uuid +import json +import click +import zipfile +import threading +import Queue +from django.db import connection +from sentry.runner.decorators import configuration + + +SHUTDOWN = object() + + +def load_bundle(q, uuid, data, sdk_info): + from sentry.models import DSymBundle, DSymSDK + + sdk = DSymSDK.objects.get_or_create( + dsym_type=sdk_info['dsym_type'], + sdk_name=sdk_info['sdk_name'], + version_major=sdk_info['version_major'], + version_minor=sdk_info['version_minor'], + version_patchlevel=sdk_info['version_patchlevel'], + version_build=sdk_info['version_build'], + )[0] + + bundle = DSymBundle.objects.get_or_create( + sdk=sdk, + cpu_name=data['arch'], + object_path=data['image'], + uuid=str(uuid), + )[0] + + step = 4000 + symbols = data['symbols'] + for idx in xrange(0, len(symbols) + step, step): + end_idx = min(idx + step, len(symbols)) + yield [{ + 'bundle_id': bundle.id, + 'address': symbols[x][0], + 'symbol': symbols[x][1], + } for x in xrange(idx, end_idx)] + + +def process_archive(members, zip, sdk_info, threads): + q = Queue.Queue(threads) + + def process_items(): + cur = connection.cursor() + cur.execute('begin') + cur.execute(''' + prepare add_sym(bigint, bigint, text) as + insert into sentry_dsymsymbol (bundle_id, address, symbol) + select $1, $2, $3 + where not exists (select 1 from sentry_dsymsymbol + where bundle_id = $1 and address = $2); + ''') + while 1: + items = q.get() + if items is SHUTDOWN: + break + cur.executemany(''' + execute add_sym(%(bundle_id)s, %(address)s, %(symbol)s); + ''', items) + cur.execute('commit') + + pool = [] + for x in xrange(threads): + t = threading.Thread(target=process_items) + t.setDaemon(True) + t.start() + pool.append(t) + + for member in members: + try: + id = uuid.UUID(member) + except ValueError: + continue + for chunk in load_bundle(q.put, id, json.load(zip.open(member)), + sdk_info): + q.put(chunk) + + for t in pool: + q.put(SHUTDOWN) + for t in pool: + t.join() + + +@click.command(name='import-system-symbols', + short_help='Import system debug symbols.') +@click.argument('bundles', type=click.Path(), nargs=-1) +@click.option('--sdk', default='iOS', help='The SDK identifier') +@click.option('--dsym-type', default='macho', help='The type of the symbol') +@click.option('--threads', default=8, help='The number of threads to use') +@configuration +def import_system_symbols(bundles, sdk, dsym_type, threads): + """Imports system symbols from preprocessed zip files into Sentry. + + It takes a list of zip files as arguments that contain preprocessed + system symbol information. These zip files contain JSON dumps. The + actual zipped up dsym files cannot be used here, they need to be + preprocessed. + """ + for path in bundles: + with zipfile.ZipFile(path) as f: + sdk_info = json.load(f.open('sdk_info')) + sdk_info['sdk_name'] = sdk + sdk_info['dsym_type'] = dsym_type + label = ('%s.%s.%s (%s)' % ( + sdk_info['version_major'], + sdk_info['version_minor'], + sdk_info['version_patchlevel'], + sdk_info['version_build'], + )).ljust(18) + with click.progressbar(f.namelist(), label=label) as bar: + process_archive(bar, f, sdk_info, threads) From 4a770f2ecf55770d0297b882b993095149cb6757 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 7 Apr 2016 22:43:47 +0200 Subject: [PATCH 02/17] Implement basic system symbol lookup from database --- src/sentry/lang/native/plugin.py | 8 +- src/sentry/lang/native/symbolizer.py | 85 +++++++++++++++++++ ..._dsymsymbol_object_address__add_dsymsd.py} | 49 +++++++---- src/sentry/models/dsymfile.py | 21 +++-- .../runner/commands/import_system_symbols.py | 32 ++++--- 5 files changed, 160 insertions(+), 35 deletions(-) rename src/sentry/migrations/{0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py => 0246_auto__add_dsymsymbol__add_unique_dsymsymbol_object_address__add_dsymsd.py} (96%) diff --git a/src/sentry/lang/native/plugin.py b/src/sentry/lang/native/plugin.py index 51337b16a52efd..d236445192aa38 100644 --- a/src/sentry/lang/native/plugin.py +++ b/src/sentry/lang/native/plugin.py @@ -4,7 +4,7 @@ from sentry.models import Project from sentry.plugins import Plugin2 -from sentry.lang.native.symbolizer import make_symbolizer, have_symsynd +from sentry.lang.native.symbolizer import Symbolizer, have_symsynd def exception_from_apple_error_or_diagnosis(error, diagnosis=None): @@ -112,9 +112,9 @@ def preprocess_apple_crash_event(data): if crashed_thread is None: return - sym = make_symbolizer(project, crash_report['binary_images'], - threads=[crashed_thread]) - with sym.driver: + sym = Symbolizer(project, crash_report['binary_images'], + threads=[crashed_thread]) + with sym: bt = sym.symbolize_backtrace(crashed_thread['backtrace']['contents']) inject_apple_backtrace(data, bt, crash.get('diagnosis'), crash.get('error'), crash_report.get('system')) diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py index 61b9e458970854..3b452c2fdfd3a4 100644 --- a/src/sentry/lang/native/symbolizer.py +++ b/src/sentry/lang/native/symbolizer.py @@ -1,14 +1,64 @@ try: from symsynd.driver import Driver from symsynd.report import ReportSymbolizer + from symsynd.macho.arch import get_cpu_name have_symsynd = True except ImportError: have_symsynd = False +from django.db import connection from sentry import options from sentry.lang.native.dsymcache import dsymcache +def find_system_symbol(img, instruction_addr): + """Finds a system symbol.""" + addr = instruction_addr - img['image_addr'] + + uuid = img['uuid'].lower() + # TODO(mitsuhiko): also funnel in version info so we can fuzzy match + cur = connection.cursor() + try: + # First try: exact match on uuid + cur.execute(''' + select symbol + from sentry_dsymsymbol s, + sentry_dsymobject o + where o.uuid = %s and + s.object_id = o.id and + s.address <= o.vmaddr + %s and + s.address >= o.vmaddr + order by address desc + limit 1; + ''', [uuid, addr]) + rv = cur.fetchone() + if rv: + return rv[0] + + # Second try: exact match on path and arch + cpu_name = get_cpu_name(img['cpu_type'], + img['cpu_subtype']) + if cpu_name is None: + return + cur.execute(''' + select symbol + from sentry_dsymsymbol s, + sentry_dsymobject o + where o.cpu_name = %s and + o.object_path = %s and + s.object_id = o.id and + s.address <= o.vmaddr + %s and + s.address >= o.vmaddr + order by address desc + limit 1; + ''', [cpu_name, img['name'], addr]) + rv = cur.fetchone() + if rv: + return rv[0] + finally: + cur.close() + + def make_symbolizer(project, binary_images, threads=None): """Creates a symbolizer for the given project and binary images. If a list of threads is referenced (from an apple crash report) then only @@ -35,3 +85,38 @@ def make_symbolizer(project, binary_images, threads=None): dsym_paths, loaded = dsymcache.fetch_dsyms(project, to_load) return ReportSymbolizer(driver, dsym_paths, binary_images) + + +class Symbolizer(object): + + def __init__(self, project, binary_images, threads=None): + self.symsynd_symbolizer = make_symbolizer(project, binary_images, + threads=threads) + self.images = dict((img['image_addr'], img) for img in binary_images) + + def __enter__(self): + return self.symsynd_symbolizer.driver.__enter__() + + def __exit__(self, *args): + return self.symsynd_symbolizer.driver.__exit__(*args) + + def symbolize_frame(self, frame): + # Step one: try to symbolize with cached dsym files. + new_frame = self.symsynd_symbolizer.symbolize_frame(frame) + if new_frame is not None: + return new_frame + + # If that does not work, look up system symbols. + img = self.images.get(frame['object_addr']) + if img is not None: + symbol = find_system_symbol(img, frame['instruction_addr']) + if symbol is not None: + return dict(frame, symbol_name=symbol, filename=None, + line=0, column=0, uuid=img['uuid']) + + def symbolize_backtrace(self, backtrace): + rv = [] + for frame in backtrace: + new_frame = self.symbolize_frame(frame) + rv.append(new_frame or frame) + return rv diff --git a/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py b/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_object_address__add_dsymsd.py similarity index 96% rename from src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py rename to src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_object_address__add_dsymsd.py index 49f6317da8f31c..e2086a7f4be2bc 100644 --- a/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_bundle_address__add_dsymsd.py +++ b/src/sentry/migrations/0246_auto__add_dsymsymbol__add_unique_dsymsymbol_object_address__add_dsymsd.py @@ -11,14 +11,14 @@ def forwards(self, orm): # Adding model 'DSymSymbol' db.create_table('sentry_dsymsymbol', ( ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), - ('bundle', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymBundle'])), - ('address', self.gf('sentry.db.models.fields.bounded.BoundedBigIntegerField')()), + ('object', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymObject'])), + ('address', self.gf('sentry.db.models.fields.bounded.BoundedBigIntegerField')(db_index=True)), ('symbol', self.gf('django.db.models.fields.TextField')()), )) db.send_create_signal('sentry', ['DSymSymbol']) - # Adding unique constraint on 'DSymSymbol', fields ['bundle', 'address'] - db.create_unique('sentry_dsymsymbol', ['bundle_id', 'address']) + # Adding unique constraint on 'DSymSymbol', fields ['object', 'address'] + db.create_unique('sentry_dsymsymbol', ['object_id', 'address']) # Adding model 'DSymSDK' db.create_table('sentry_dsymsdk', ( @@ -35,13 +35,22 @@ def forwards(self, orm): # Adding index on 'DSymSDK', fields ['version_major', 'version_minor', 'version_patchlevel', 'version_build'] db.create_index('sentry_dsymsdk', ['version_major', 'version_minor', 'version_patchlevel', 'version_build']) - # Adding model 'DSymBundle' - db.create_table('sentry_dsymbundle', ( + # Adding model 'DSymObject' + db.create_table('sentry_dsymobject', ( ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), - ('sdk', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymSDK'])), ('cpu_name', self.gf('django.db.models.fields.CharField')(max_length=40)), ('object_path', self.gf('django.db.models.fields.TextField')(db_index=True)), ('uuid', self.gf('django.db.models.fields.CharField')(max_length=36, db_index=True)), + ('vmaddr', self.gf('sentry.db.models.fields.bounded.BoundedBigIntegerField')(null=True)), + ('vmsize', self.gf('sentry.db.models.fields.bounded.BoundedBigIntegerField')(null=True)), + )) + db.send_create_signal('sentry', ['DSymObject']) + + # Adding model 'DSymBundle' + db.create_table('sentry_dsymbundle', ( + ('id', self.gf('sentry.db.models.fields.bounded.BoundedBigAutoField')(primary_key=True)), + ('sdk', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymSDK'])), + ('object', self.gf('sentry.db.models.fields.foreignkey.FlexibleForeignKey')(to=orm['sentry.DSymObject'])), )) db.send_create_signal('sentry', ['DSymBundle']) @@ -50,8 +59,8 @@ def backwards(self, orm): # Removing index on 'DSymSDK', fields ['version_major', 'version_minor', 'version_patchlevel', 'version_build'] db.delete_index('sentry_dsymsdk', ['version_major', 'version_minor', 'version_patchlevel', 'version_build']) - # Removing unique constraint on 'DSymSymbol', fields ['bundle', 'address'] - db.delete_unique('sentry_dsymsymbol', ['bundle_id', 'address']) + # Removing unique constraint on 'DSymSymbol', fields ['object', 'address'] + db.delete_unique('sentry_dsymsymbol', ['object_id', 'address']) # Deleting model 'DSymSymbol' db.delete_table('sentry_dsymsymbol') @@ -59,6 +68,9 @@ def backwards(self, orm): # Deleting model 'DSymSDK' db.delete_table('sentry_dsymsdk') + # Deleting model 'DSymObject' + db.delete_table('sentry_dsymobject') + # Deleting model 'DSymBundle' db.delete_table('sentry_dsymbundle') @@ -128,7 +140,7 @@ def backwards(self, orm): 'sentry.broadcast': { 'Meta': {'object_name': 'Broadcast'}, 'date_added': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'date_expires': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2016, 4, 13, 0, 0)', 'null': 'True', 'blank': 'True'}), + 'date_expires': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2016, 4, 14, 0, 0)', 'null': 'True', 'blank': 'True'}), 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}), 'link': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), @@ -151,11 +163,18 @@ def backwards(self, orm): }, 'sentry.dsymbundle': { 'Meta': {'object_name': 'DSymBundle'}, + 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'object': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymObject']"}), + 'sdk': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymSDK']"}) + }, + 'sentry.dsymobject': { + 'Meta': {'object_name': 'DSymObject'}, 'cpu_name': ('django.db.models.fields.CharField', [], {'max_length': '40'}), 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), 'object_path': ('django.db.models.fields.TextField', [], {'db_index': 'True'}), - 'sdk': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymSDK']"}), - 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '36', 'db_index': 'True'}) + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '36', 'db_index': 'True'}), + 'vmaddr': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True'}), + 'vmsize': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'null': 'True'}) }, 'sentry.dsymsdk': { 'Meta': {'object_name': 'DSymSDK', 'index_together': "[('version_major', 'version_minor', 'version_patchlevel', 'version_build')]"}, @@ -168,10 +187,10 @@ def backwards(self, orm): 'version_patchlevel': ('django.db.models.fields.IntegerField', [], {}) }, 'sentry.dsymsymbol': { - 'Meta': {'unique_together': "[('bundle', 'address')]", 'object_name': 'DSymSymbol'}, - 'address': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {}), - 'bundle': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymBundle']"}), + 'Meta': {'unique_together': "[('object', 'address')]", 'object_name': 'DSymSymbol'}, + 'address': ('sentry.db.models.fields.bounded.BoundedBigIntegerField', [], {'db_index': 'True'}), 'id': ('sentry.db.models.fields.bounded.BoundedBigAutoField', [], {'primary_key': 'True'}), + 'object': ('sentry.db.models.fields.foreignkey.FlexibleForeignKey', [], {'to': "orm['sentry.DSymObject']"}), 'symbol': ('django.db.models.fields.TextField', [], {}) }, 'sentry.event': { diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index 1419ca740fe0fc..2b14bac3ea78dd 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -15,7 +15,7 @@ from django.db import models, transaction, IntegrityError try: - from symsynd.mach import get_macho_uuids + from symsynd.macho.arch import get_macho_uuids have_symsynd = True except ImportError: have_symsynd = False @@ -74,12 +74,23 @@ class Meta: ] -class DSymBundle(Model): +class DSymObject(Model): __core__ = False - sdk = FlexibleForeignKey('sentry.DSymSDK') cpu_name = models.CharField(max_length=40) object_path = models.TextField(db_index=True) uuid = models.CharField(max_length=36, db_index=True) + vmaddr = BoundedBigIntegerField(null=True) + vmsize = BoundedBigIntegerField(null=True) + + class Meta: + app_label = 'sentry' + db_table = 'sentry_dsymobject' + + +class DSymBundle(Model): + __core__ = False + sdk = FlexibleForeignKey('sentry.DSymSDK') + object = FlexibleForeignKey('sentry.DSymObject') class Meta: app_label = 'sentry' @@ -88,7 +99,7 @@ class Meta: class DSymSymbol(Model): __core__ = False - bundle = FlexibleForeignKey('sentry.DSymBundle') + object = FlexibleForeignKey('sentry.DSymObject') address = BoundedBigIntegerField(db_index=True) symbol = models.TextField() @@ -96,7 +107,7 @@ class Meta: app_label = 'sentry' db_table = 'sentry_dsymsymbol' unique_together = [ - ('bundle', 'address'), + ('object', 'address'), ] diff --git a/src/sentry/runner/commands/import_system_symbols.py b/src/sentry/runner/commands/import_system_symbols.py index 9e73c882f6a94a..ed49aa81e1278d 100644 --- a/src/sentry/runner/commands/import_system_symbols.py +++ b/src/sentry/runner/commands/import_system_symbols.py @@ -13,7 +13,7 @@ import zipfile import threading import Queue -from django.db import connection +from django.db import connection, IntegrityError from sentry.runner.decorators import configuration @@ -21,7 +21,7 @@ def load_bundle(q, uuid, data, sdk_info): - from sentry.models import DSymBundle, DSymSDK + from sentry.models import DSymBundle, DSymObject, DSymSDK sdk = DSymSDK.objects.get_or_create( dsym_type=sdk_info['dsym_type'], @@ -32,11 +32,17 @@ def load_bundle(q, uuid, data, sdk_info): version_build=sdk_info['version_build'], )[0] - bundle = DSymBundle.objects.get_or_create( - sdk=sdk, + obj = DSymObject.objects.get_or_create( cpu_name=data['arch'], - object_path=data['image'], + object_path='/' + data['image'].strip('/'), uuid=str(uuid), + vmaddr=data['vmaddr'], + vmsize=data['vmsize'], + )[0] + + DSymBundle.objects.get_or_create( + sdk=sdk, + object=obj )[0] step = 4000 @@ -44,7 +50,7 @@ def load_bundle(q, uuid, data, sdk_info): for idx in xrange(0, len(symbols) + step, step): end_idx = min(idx + step, len(symbols)) yield [{ - 'bundle_id': bundle.id, + 'object_id': obj.id, 'address': symbols[x][0], 'symbol': symbols[x][1], } for x in xrange(idx, end_idx)] @@ -58,18 +64,22 @@ def process_items(): cur.execute('begin') cur.execute(''' prepare add_sym(bigint, bigint, text) as - insert into sentry_dsymsymbol (bundle_id, address, symbol) + insert into sentry_dsymsymbol (object_id, address, symbol) select $1, $2, $3 where not exists (select 1 from sentry_dsymsymbol - where bundle_id = $1 and address = $2); + where object_id = $1 and address = $2); ''') while 1: items = q.get() if items is SHUTDOWN: break - cur.executemany(''' - execute add_sym(%(bundle_id)s, %(address)s, %(symbol)s); - ''', items) + try: + cur.executemany(''' + execute add_sym(%(object_id)s, %(address)s, %(symbol)s); + ''', items) + except IntegrityError: + print 'warning: already seen' + continue cur.execute('commit') pool = [] From 92d49a6f0ae05cb561f328cdd1d5a38a12571a0e Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 9 Apr 2016 20:15:27 +0200 Subject: [PATCH 03/17] Roll back on duplicate failure in import script --- .../runner/commands/import_system_symbols.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/sentry/runner/commands/import_system_symbols.py b/src/sentry/runner/commands/import_system_symbols.py index ed49aa81e1278d..da35d4547f726d 100644 --- a/src/sentry/runner/commands/import_system_symbols.py +++ b/src/sentry/runner/commands/import_system_symbols.py @@ -73,13 +73,15 @@ def process_items(): items = q.get() if items is SHUTDOWN: break - try: - cur.executemany(''' - execute add_sym(%(object_id)s, %(address)s, %(symbol)s); - ''', items) - except IntegrityError: - print 'warning: already seen' - continue + while 1: + try: + cur.executemany(''' + execute add_sym(%(object_id)s, %(address)s, %(symbol)s); + ''', items) + except IntegrityError: + connection.rollback() + continue + break cur.execute('commit') pool = [] From c4b048859ccfc7dc1f4c8e7ba3eec60484fd9c74 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 12 Apr 2016 20:11:50 +0200 Subject: [PATCH 04/17] Added fuzzy matching --- src/sentry/lang/native/plugin.py | 6 ++- src/sentry/lang/native/symbolizer.py | 59 ++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/sentry/lang/native/plugin.py b/src/sentry/lang/native/plugin.py index d236445192aa38..9df9643859d9f3 100644 --- a/src/sentry/lang/native/plugin.py +++ b/src/sentry/lang/native/plugin.py @@ -112,12 +112,14 @@ def preprocess_apple_crash_event(data): if crashed_thread is None: return + system = crash_report.get('system') sym = Symbolizer(project, crash_report['binary_images'], threads=[crashed_thread]) with sym: - bt = sym.symbolize_backtrace(crashed_thread['backtrace']['contents']) + bt = sym.symbolize_backtrace(crashed_thread['backtrace']['contents'], + system) inject_apple_backtrace(data, bt, crash.get('diagnosis'), - crash.get('error'), crash_report.get('system')) + crash.get('error'), system) return data diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py index 3b452c2fdfd3a4..3f2f540ca14080 100644 --- a/src/sentry/lang/native/symbolizer.py +++ b/src/sentry/lang/native/symbolizer.py @@ -11,12 +11,35 @@ from sentry.lang.native.dsymcache import dsymcache -def find_system_symbol(img, instruction_addr): +SDK_MAPPING = { + 'iPhone OS': 'iOS', +} + + +def get_sdk_from_system_info(info): + if not info: + return None + try: + sdk_name = SDK_MAPPING[info['system_name']] + system_version = tuple(int(x) for x in ( + info['system_version'] + '.0' * 3).split('.')[:3]) + except LookupError: + return None + + return { + 'dsym_type': 'macho', + 'sdk_name': sdk_name, + 'version_major': system_version[0], + 'version_minor': system_version[1], + 'version_patchlevel': system_version[2], + } + + +def find_system_symbol(img, instruction_addr, system_info=None): """Finds a system symbol.""" addr = instruction_addr - img['image_addr'] uuid = img['uuid'].lower() - # TODO(mitsuhiko): also funnel in version info so we can fuzzy match cur = connection.cursor() try: # First try: exact match on uuid @@ -38,20 +61,33 @@ def find_system_symbol(img, instruction_addr): # Second try: exact match on path and arch cpu_name = get_cpu_name(img['cpu_type'], img['cpu_subtype']) - if cpu_name is None: + sdk_info = get_sdk_from_system_info(system_info) + if sdk_info is None or cpu_name is None: return + cur.execute(''' select symbol from sentry_dsymsymbol s, - sentry_dsymobject o - where o.cpu_name = %s and - o.object_path = %s and + sentry_dsymobject o, + sentry_dsymsdk k, + sentry_dsymbundle b + where b.sdk_id = k.id and + b.object_id = o.id and s.object_id = o.id and + k.sdk_name = %s and + k.dsym_type = %s and + k.version_major = %s and + k.version_minor = %s and + k.version_patchlevel = %s and + o.cpu_name = %s and + o.object_path = %s and s.address <= o.vmaddr + %s and s.address >= o.vmaddr order by address desc limit 1; - ''', [cpu_name, img['name'], addr]) + ''', [sdk_info['sdk_name'], sdk_info['dsym_type'], + sdk_info['version_major'], sdk_info['version_minor'], + sdk_info['version_patchlevel'], cpu_name, img['name'], addr]) rv = cur.fetchone() if rv: return rv[0] @@ -100,7 +136,7 @@ def __enter__(self): def __exit__(self, *args): return self.symsynd_symbolizer.driver.__exit__(*args) - def symbolize_frame(self, frame): + def symbolize_frame(self, frame, system_info=None): # Step one: try to symbolize with cached dsym files. new_frame = self.symsynd_symbolizer.symbolize_frame(frame) if new_frame is not None: @@ -109,14 +145,15 @@ def symbolize_frame(self, frame): # If that does not work, look up system symbols. img = self.images.get(frame['object_addr']) if img is not None: - symbol = find_system_symbol(img, frame['instruction_addr']) + symbol = find_system_symbol(img, frame['instruction_addr'], + system_info) if symbol is not None: return dict(frame, symbol_name=symbol, filename=None, line=0, column=0, uuid=img['uuid']) - def symbolize_backtrace(self, backtrace): + def symbolize_backtrace(self, backtrace, system_info=None): rv = [] for frame in backtrace: - new_frame = self.symbolize_frame(frame) + new_frame = self.symbolize_frame(frame, system_info) rv.append(new_frame or frame) return rv From bb3a287fa677b045b809f4f04cb9297dc4615142 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 12 Apr 2016 20:35:31 +0200 Subject: [PATCH 05/17] Make some import local --- src/sentry/runner/commands/import_system_symbols.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/runner/commands/import_system_symbols.py b/src/sentry/runner/commands/import_system_symbols.py index da35d4547f726d..b43af350145eb3 100644 --- a/src/sentry/runner/commands/import_system_symbols.py +++ b/src/sentry/runner/commands/import_system_symbols.py @@ -10,9 +10,7 @@ import uuid import json import click -import zipfile import threading -import Queue from django.db import connection, IntegrityError from sentry.runner.decorators import configuration @@ -57,6 +55,7 @@ def load_bundle(q, uuid, data, sdk_info): def process_archive(members, zip, sdk_info, threads): + import Queue q = Queue.Queue(threads) def process_items(): @@ -121,6 +120,7 @@ def import_system_symbols(bundles, sdk, dsym_type, threads): actual zipped up dsym files cannot be used here, they need to be preprocessed. """ + import zipfile for path in bundles: with zipfile.ZipFile(path) as f: sdk_info = json.load(f.open('sdk_info')) From 47cd67c8c82d7574ea6d6f5a03ad91e5cc9b7e5b Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 12 Apr 2016 20:47:40 +0200 Subject: [PATCH 06/17] Bump symsynd to 0.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ebe63b0a5d238f..7c5e2afbad9f2f 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ ] dsym_requires = [ - 'symsynd>=0.3.0,<1.0.0', + 'symsynd>=0.4.0,<1.0.0', ] From c848a78756a98f09e7959a034f0fba958275711b Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 13 Apr 2016 12:09:47 +0200 Subject: [PATCH 07/17] Added demangling for system frames and trim symbols --- setup.py | 2 +- src/sentry/lang/native/symbolizer.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 7c5e2afbad9f2f..b22dd6344b3006 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ ] dsym_requires = [ - 'symsynd>=0.4.0,<1.0.0', + 'symsynd>=0.5.0,<1.0.0', ] diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py index 3f2f540ca14080..c090b7e4a396db 100644 --- a/src/sentry/lang/native/symbolizer.py +++ b/src/sentry/lang/native/symbolizer.py @@ -2,6 +2,7 @@ from symsynd.driver import Driver from symsynd.report import ReportSymbolizer from symsynd.macho.arch import get_cpu_name + from symsynd.demangle import demangle_symbol have_symsynd = True except ImportError: have_symsynd = False @@ -9,6 +10,7 @@ from django.db import connection from sentry import options from sentry.lang.native.dsymcache import dsymcache +from sentry.utils.safe import trim SDK_MAPPING = { @@ -16,6 +18,12 @@ } +def trim_frame(frame): + frame['symbol_name'] = trim(frame.get('symbol_name'), 1024) + frame['filename'] = trim(frame.get('filename'), 512) + return frame + + def get_sdk_from_system_info(info): if not info: return None @@ -140,7 +148,7 @@ def symbolize_frame(self, frame, system_info=None): # Step one: try to symbolize with cached dsym files. new_frame = self.symsynd_symbolizer.symbolize_frame(frame) if new_frame is not None: - return new_frame + return trim_frame(new_frame) # If that does not work, look up system symbols. img = self.images.get(frame['object_addr']) @@ -148,8 +156,10 @@ def symbolize_frame(self, frame, system_info=None): symbol = find_system_symbol(img, frame['instruction_addr'], system_info) if symbol is not None: - return dict(frame, symbol_name=symbol, filename=None, - line=0, column=0, uuid=img['uuid']) + symbol = demangle_symbol(symbol) or symbol + rv = dict(frame, symbol_name=symbol, filename=None, + line=0, column=0, uuid=img['uuid']) + return trim_frame(rv) def symbolize_backtrace(self, backtrace, system_info=None): rv = [] From b22a3641c9dcd66d361c8ebfc57d13d25e794618 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 13 Apr 2016 12:16:13 +0200 Subject: [PATCH 08/17] Bump symsynd dependency to 0.5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b22dd6344b3006..a90a59b78a2149 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ ] dsym_requires = [ - 'symsynd>=0.5.0,<1.0.0', + 'symsynd>=0.5.2,<1.0.0', ] From 78b751e831230f7116faf3bf3aaf9b0e4b74f863 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 13 Apr 2016 21:12:32 +0200 Subject: [PATCH 09/17] Match lengths in stacktrace --- src/sentry/lang/native/symbolizer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py index c090b7e4a396db..4752a86848699d 100644 --- a/src/sentry/lang/native/symbolizer.py +++ b/src/sentry/lang/native/symbolizer.py @@ -19,8 +19,9 @@ def trim_frame(frame): - frame['symbol_name'] = trim(frame.get('symbol_name'), 1024) - frame['filename'] = trim(frame.get('filename'), 512) + # This matches what's in stacktrace.py + frame['symbol_name'] = trim(frame.get('symbol_name'), 256) + frame['filename'] = trim(frame.get('filename'), 256) return frame From 80d392f2e51d8cb18e45616a27dfd90cfa8c14f7 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 14 Apr 2016 15:15:51 +0200 Subject: [PATCH 10/17] Refactored system symbol code and optimized storage in postres --- setup.py | 2 +- src/sentry/lang/native/symbolizer.py | 94 +------- src/sentry/models/dsymfile.py | 161 ++++++++++++-- src/sentry/runner/__init__.py | 2 +- src/sentry/runner/commands/dsym.py | 207 ++++++++++++++++++ .../runner/commands/import_system_symbols.py | 136 ------------ 6 files changed, 359 insertions(+), 243 deletions(-) create mode 100644 src/sentry/runner/commands/dsym.py delete mode 100644 src/sentry/runner/commands/import_system_symbols.py diff --git a/setup.py b/setup.py index a90a59b78a2149..df8d548e9cf6d9 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ ] dsym_requires = [ - 'symsynd>=0.5.2,<1.0.0', + 'symsynd>=0.6.0,<1.0.0', ] diff --git a/src/sentry/lang/native/symbolizer.py b/src/sentry/lang/native/symbolizer.py index 4752a86848699d..c3d4b5b2b3c276 100644 --- a/src/sentry/lang/native/symbolizer.py +++ b/src/sentry/lang/native/symbolizer.py @@ -7,101 +7,31 @@ except ImportError: have_symsynd = False -from django.db import connection from sentry import options from sentry.lang.native.dsymcache import dsymcache from sentry.utils.safe import trim - - -SDK_MAPPING = { - 'iPhone OS': 'iOS', -} +from sentry.models import DSymSymbol +from sentry.models.dsymfile import MAX_SYM def trim_frame(frame): # This matches what's in stacktrace.py - frame['symbol_name'] = trim(frame.get('symbol_name'), 256) + frame['symbol_name'] = trim(frame.get('symbol_name'), MAX_SYM) frame['filename'] = trim(frame.get('filename'), 256) return frame -def get_sdk_from_system_info(info): - if not info: - return None - try: - sdk_name = SDK_MAPPING[info['system_name']] - system_version = tuple(int(x) for x in ( - info['system_version'] + '.0' * 3).split('.')[:3]) - except LookupError: - return None - - return { - 'dsym_type': 'macho', - 'sdk_name': sdk_name, - 'version_major': system_version[0], - 'version_minor': system_version[1], - 'version_patchlevel': system_version[2], - } - - def find_system_symbol(img, instruction_addr, system_info=None): """Finds a system symbol.""" - addr = instruction_addr - img['image_addr'] - - uuid = img['uuid'].lower() - cur = connection.cursor() - try: - # First try: exact match on uuid - cur.execute(''' - select symbol - from sentry_dsymsymbol s, - sentry_dsymobject o - where o.uuid = %s and - s.object_id = o.id and - s.address <= o.vmaddr + %s and - s.address >= o.vmaddr - order by address desc - limit 1; - ''', [uuid, addr]) - rv = cur.fetchone() - if rv: - return rv[0] - - # Second try: exact match on path and arch - cpu_name = get_cpu_name(img['cpu_type'], - img['cpu_subtype']) - sdk_info = get_sdk_from_system_info(system_info) - if sdk_info is None or cpu_name is None: - return - - cur.execute(''' - select symbol - from sentry_dsymsymbol s, - sentry_dsymobject o, - sentry_dsymsdk k, - sentry_dsymbundle b - where b.sdk_id = k.id and - b.object_id = o.id and - s.object_id = o.id and - k.sdk_name = %s and - k.dsym_type = %s and - k.version_major = %s and - k.version_minor = %s and - k.version_patchlevel = %s and - o.cpu_name = %s and - o.object_path = %s and - s.address <= o.vmaddr + %s and - s.address >= o.vmaddr - order by address desc - limit 1; - ''', [sdk_info['sdk_name'], sdk_info['dsym_type'], - sdk_info['version_major'], sdk_info['version_minor'], - sdk_info['version_patchlevel'], cpu_name, img['name'], addr]) - rv = cur.fetchone() - if rv: - return rv[0] - finally: - cur.close() + return DSymSymbol.objects.lookup_symbol( + instruction_addr=instruction_addr, + image_addr=img['image_addr'], + uuid=img['uuid'], + cpu_name=get_cpu_name(img['cpu_type'], + img['cpu_subtype']), + object_path=img['name'], + system_info=system_info + ) def make_symbolizer(project, binary_images, threads=None): diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index 2b14bac3ea78dd..34b6c1105337c3 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -12,7 +12,7 @@ import shutil import hashlib import tempfile -from django.db import models, transaction, IntegrityError +from django.db import models, transaction, connection, IntegrityError try: from symsynd.macho.arch import get_macho_uuids @@ -21,39 +21,87 @@ have_symsynd = False from sentry.db.models import FlexibleForeignKey, Model, BoundedBigIntegerField, \ - sane_repr + sane_repr, BaseManager from sentry.models.file import File from sentry.utils.zip import safe_extract_zip +MAX_SYM = 256 KNOWN_DSYM_TYPES = { 'application/x-mach-binary': 'macho' } +SDK_MAPPING = { + 'iPhone OS': 'iOS', +} -''' -system symbols: - - architecture VARCHAR(20) - sdk-version MAJOR.MINOR.PATCHLEVEL-BUILD - uuid UUID - path VARCHAR(255) - address BIGINT - symbol TEXT - -lookup logic: - primary lookup: - uuid -> exact match - address -> lower than or equal to reference address - limit 1 +def get_sdk_from_system_info(info): + if not info: + return None + try: + sdk_name = SDK_MAPPING[info['system_name']] + system_version = tuple(int(x) for x in ( + info['system_version'] + '.0' * 3).split('.')[:3]) + except LookupError: + return None - secondary lookup: - path -> exact match - sdk-version -> fuzzy match - address -> lower than or equal to reference address - limit 1 -''' + return { + 'dsym_type': 'macho', + 'sdk_name': sdk_name, + 'version_major': system_version[0], + 'version_minor': system_version[1], + 'version_patchlevel': system_version[2], + } + + +class DSymSDKManager(BaseManager): + + def enumerate_sdks(self, sdk=None, version=None): + """Return a grouped list of SDKs.""" + filter = '' + args = [] + if version is not None: + for col, val in zip(['major', 'minor', 'patchlevel'], + version.split('.')): + if not val.isdigit(): + return [] + filter += ' and k.version_%s = %d' % ( + col, + int(val) + ) + if sdk is not None: + filter += ' and k.sdk_name = %s' + args.append(sdk) + cur = connection.cursor() + cur.execute(''' + select distinct k.*, count(b.*) as bundle_count, o.cpu_name + from sentry_dsymsdk k, + sentry_dsymbundle b, + sentry_dsymobject o + where b.sdk_id = k.id and + b.object_id = o.id %s + group by k.id, k.sdk_name, o.cpu_name + ''' % filter, args) + rv = [] + for row in cur.fetchall(): + row = dict(zip([x[0] for x in cur.description], row)) + ver = '%s.%s.%s' % ( + row['version_major'], + row['version_minor'], + row['version_patchlevel'] + ) + rv.append({ + 'sdk_name': row['sdk_name'], + 'version': ver, + 'build': row['version_build'], + 'bundle_count': row['bundle_count'], + 'cpu_name': row['cpu_name'], + }) + return sorted(rv, key=lambda x: (x['sdk_name'], + x['version'], + x['build'], + x['cpu_name'])) class DSymSDK(Model): @@ -65,6 +113,8 @@ class DSymSDK(Model): version_patchlevel = models.IntegerField() version_build = models.CharField(max_length=40) + objects = DSymSDKManager() + class Meta: app_label = 'sentry' db_table = 'sentry_dsymsdk' @@ -97,12 +147,77 @@ class Meta: db_table = 'sentry_dsymbundle' +class DSymSymbolManager(BaseManager): + + def lookup_symbol(self, instruction_addr, image_addr, uuid, + cpu_name=None, object_path=None, system_info=None): + """Finds a system symbol.""" + addr = instruction_addr - image_addr + + uuid = str(uuid).lower() + cur = connection.cursor() + try: + # First try: exact match on uuid + cur.execute(''' + select symbol + from sentry_dsymsymbol s, + sentry_dsymobject o + where o.uuid = %s and + s.object_id = o.id and + s.address <= o.vmaddr + %s and + s.address >= o.vmaddr + order by address desc + limit 1; + ''', [uuid, addr]) + rv = cur.fetchone() + if rv: + return rv[0] + + # Second try: exact match on path and arch + sdk_info = get_sdk_from_system_info(system_info) + if sdk_info is None or \ + cpu_name is None or \ + object_path is None: + return + + cur.execute(''' + select symbol + from sentry_dsymsymbol s, + sentry_dsymobject o, + sentry_dsymsdk k, + sentry_dsymbundle b + where b.sdk_id = k.id and + b.object_id = o.id and + s.object_id = o.id and + k.sdk_name = %s and + k.dsym_type = %s and + k.version_major = %s and + k.version_minor = %s and + k.version_patchlevel = %s and + o.cpu_name = %s and + o.object_path = %s and + s.address <= o.vmaddr + %s and + s.address >= o.vmaddr + order by address desc + limit 1; + ''', [sdk_info['sdk_name'], sdk_info['dsym_type'], + sdk_info['version_major'], sdk_info['version_minor'], + sdk_info['version_patchlevel'], cpu_name, object_path, addr]) + rv = cur.fetchone() + if rv: + return rv[0] + finally: + cur.close() + + class DSymSymbol(Model): __core__ = False object = FlexibleForeignKey('sentry.DSymObject') address = BoundedBigIntegerField(db_index=True) symbol = models.TextField() + objects = DSymSymbolManager() + class Meta: app_label = 'sentry' db_table = 'sentry_dsymsymbol' diff --git a/src/sentry/runner/__init__.py b/src/sentry/runner/__init__.py index 97d2f46e820c36..e2ce1f0130b32e 100755 --- a/src/sentry/runner/__init__.py +++ b/src/sentry/runner/__init__.py @@ -61,7 +61,7 @@ def cli(ctx, config): 'sentry.runner.commands.start.start', 'sentry.runner.commands.tsdb.tsdb', 'sentry.runner.commands.upgrade.upgrade', - 'sentry.runner.commands.import_system_symbols.import_system_symbols', + 'sentry.runner.commands.dsym.dsym', )) diff --git a/src/sentry/runner/commands/dsym.py b/src/sentry/runner/commands/dsym.py new file mode 100644 index 00000000000000..8228ed0188ce8c --- /dev/null +++ b/src/sentry/runner/commands/dsym.py @@ -0,0 +1,207 @@ +""" +sentry.runner.commands.dsym +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details. +:license: BSD, see LICENSE for more details. +""" +from __future__ import absolute_import + +import uuid +import json +import click +import threading + +from django.db import connection + +from sentry.runner.decorators import configuration + + +SHUTDOWN = object() + + +def load_bundle(q, uuid, data, sdk_info, trim_symbols, demangle): + from sentry.models import DSymBundle, DSymObject, DSymSDK + from sentry.models.dsymfile import MAX_SYM + from symsynd.demangle import demangle_symbol + + def _process_symbol(sym): + too_long = trim_symbols and len(sym) > MAX_SYM + if demangle or too_long: + new_sym = demangle_symbol(sym) + if new_sym is not None and (len(new_sym) < sym or too_long): + sym = new_sym + if trim_symbols: + sym = sym[:MAX_SYM] + return sym + + sdk = DSymSDK.objects.get_or_create( + dsym_type=sdk_info['dsym_type'], + sdk_name=sdk_info['sdk_name'], + version_major=sdk_info['version_major'], + version_minor=sdk_info['version_minor'], + version_patchlevel=sdk_info['version_patchlevel'], + version_build=sdk_info['version_build'], + )[0] + + obj = DSymObject.objects.get_or_create( + cpu_name=data['arch'], + object_path='/' + data['image'].strip('/'), + uuid=str(uuid), + vmaddr=data['vmaddr'], + vmsize=data['vmsize'], + )[0] + + DSymBundle.objects.get_or_create( + sdk=sdk, + object=obj + )[0] + + step = 4000 + symbols = data['symbols'] + for idx in xrange(0, len(symbols) + step, step): + end_idx = min(idx + step, len(symbols)) + batch = {} + for x in xrange(idx, end_idx): + addr = symbols[x][0] + batch[obj.id, addr] = { + 'object_id': obj.id, + 'address': addr, + 'symbol': _process_symbol(symbols[x][1]), + } + yield sorted(batch.values(), key=lambda x: x['address']) + + +def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False, + demangle=True): + import Queue + q = Queue.Queue(threads) + + def process_items(): + cur = connection.cursor() + cur.execute('begin') + cur.execute(''' + prepare add_sym(bigint, bigint, text) as + insert into sentry_dsymsymbol (object_id, address, symbol) + select $1, $2, $3 + where not exists (select 1 from sentry_dsymsymbol + where object_id = $1 and address = $2); + ''') + while 1: + items = q.get() + if items is SHUTDOWN: + break + cur.executemany(''' + execute add_sym(%(object_id)s, %(address)s, %(symbol)s); + ''', items) + cur.execute('commit') + + pool = [] + for x in xrange(threads): + t = threading.Thread(target=process_items) + t.setDaemon(True) + t.start() + pool.append(t) + + for member in members: + try: + id = uuid.UUID(member) + except ValueError: + continue + for chunk in load_bundle(q.put, id, json.load(zip.open(member)), + sdk_info, trim_symbols, demangle): + q.put(chunk) + + for t in pool: + q.put(SHUTDOWN) + for t in pool: + t.join() + + +@click.group(name='dsym') +def dsym(): + """Manage system symbols in Sentry. + + This allows you to import and manage globally shared system symbols in + the Sentry installation. In particular this is useful for iOS where + system symbols need to be ingested before stacktraces can be fully + symbolized due to device optimizations. + """ + + +@dsym.command(name='import-system-symbols', + short_help='Import system debug symbols.') +@click.argument('bundles', type=click.Path(), nargs=-1) +@click.option('--threads', default=8, help='The number of threads to use') +@click.option('--trim-symbols', is_flag=True, + help='If enabled symbols are trimmed before storing. ' + 'This reduces the database size but means that symbols are ' + 'already trimmed on the way to the database.') +@click.option('--no-demangle', is_flag=True, + help='If this is set to true symbols are never demangled. ' + 'By default symbols are demangled if they are trimmed or ' + 'demangled symbols are shorter than mangled ones. Enabling ' + 'this option speeds up importing slightly.') +@configuration +def import_system_symbols(bundles, threads, trim_symbols, no_demangle): + """Imports system symbols from preprocessed zip files into Sentry. + + It takes a list of zip files as arguments that contain preprocessed + system symbol information. These zip files contain JSON dumps. The + actual zipped up dsym files cannot be used here, they need to be + preprocessed. + """ + import zipfile + for path in bundles: + with zipfile.ZipFile(path) as f: + sdk_info = json.load(f.open('sdk_info')) + label = ('%s.%s.%s (%s)' % ( + sdk_info['version_major'], + sdk_info['version_minor'], + sdk_info['version_patchlevel'], + sdk_info['version_build'], + )).ljust(18) + with click.progressbar(f.namelist(), label=label) as bar: + process_archive(bar, f, sdk_info, threads, + trim_symbols=trim_symbols, + demangle=not no_demangle) + + +@dsym.command(name='sdks', short_help='List SDKs') +@click.option('--sdk', help='Only include the given SDK instead of all.') +@click.option('--version', help='Optionally a version filter. For instance ' + '9 returns all versions 9.*, 9.1 returns 9.1.* etc.') +@configuration +def sdks(sdk, version): + """Print a list of all installed SDKs and a breakdown of the symbols + contained within. This queries the system symbol database and reports + all SDKs and versions that symbols exist for. The output is broken down + by minor versions, builds and cpu architectures. For each of those a + count of the stored bundles is returned. (A bundle in this case is a + single binary) + """ + from sentry.models import DSymSDK + last_prefix = None + click.secho(' %-8s %-10s %-12s %-8s %s' % ( + 'SDK', + 'Version', + 'Build', + 'CPU', + 'Bundles', + ), fg='cyan') + click.secho('-' * click.get_terminal_size()[0], fg='yellow') + for sdk in DSymSDK.objects.enumerate_sdks(sdk=sdk, version=version): + prefix = ' %-8s %-10s ' % ( + sdk['sdk_name'], + sdk['version'] + ) + if prefix == last_prefix: + prefix = ' ' * len(prefix) + else: + last_prefix = prefix + click.echo('%s%-12s %-8s %d' % ( + prefix, + sdk['build'], + sdk['cpu_name'], + sdk['bundle_count'], + )) diff --git a/src/sentry/runner/commands/import_system_symbols.py b/src/sentry/runner/commands/import_system_symbols.py deleted file mode 100644 index b43af350145eb3..00000000000000 --- a/src/sentry/runner/commands/import_system_symbols.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -sentry.runner.commands.import_system_symbols -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:copyright: (c) 2015 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" -from __future__ import absolute_import - -import uuid -import json -import click -import threading -from django.db import connection, IntegrityError -from sentry.runner.decorators import configuration - - -SHUTDOWN = object() - - -def load_bundle(q, uuid, data, sdk_info): - from sentry.models import DSymBundle, DSymObject, DSymSDK - - sdk = DSymSDK.objects.get_or_create( - dsym_type=sdk_info['dsym_type'], - sdk_name=sdk_info['sdk_name'], - version_major=sdk_info['version_major'], - version_minor=sdk_info['version_minor'], - version_patchlevel=sdk_info['version_patchlevel'], - version_build=sdk_info['version_build'], - )[0] - - obj = DSymObject.objects.get_or_create( - cpu_name=data['arch'], - object_path='/' + data['image'].strip('/'), - uuid=str(uuid), - vmaddr=data['vmaddr'], - vmsize=data['vmsize'], - )[0] - - DSymBundle.objects.get_or_create( - sdk=sdk, - object=obj - )[0] - - step = 4000 - symbols = data['symbols'] - for idx in xrange(0, len(symbols) + step, step): - end_idx = min(idx + step, len(symbols)) - yield [{ - 'object_id': obj.id, - 'address': symbols[x][0], - 'symbol': symbols[x][1], - } for x in xrange(idx, end_idx)] - - -def process_archive(members, zip, sdk_info, threads): - import Queue - q = Queue.Queue(threads) - - def process_items(): - cur = connection.cursor() - cur.execute('begin') - cur.execute(''' - prepare add_sym(bigint, bigint, text) as - insert into sentry_dsymsymbol (object_id, address, symbol) - select $1, $2, $3 - where not exists (select 1 from sentry_dsymsymbol - where object_id = $1 and address = $2); - ''') - while 1: - items = q.get() - if items is SHUTDOWN: - break - while 1: - try: - cur.executemany(''' - execute add_sym(%(object_id)s, %(address)s, %(symbol)s); - ''', items) - except IntegrityError: - connection.rollback() - continue - break - cur.execute('commit') - - pool = [] - for x in xrange(threads): - t = threading.Thread(target=process_items) - t.setDaemon(True) - t.start() - pool.append(t) - - for member in members: - try: - id = uuid.UUID(member) - except ValueError: - continue - for chunk in load_bundle(q.put, id, json.load(zip.open(member)), - sdk_info): - q.put(chunk) - - for t in pool: - q.put(SHUTDOWN) - for t in pool: - t.join() - - -@click.command(name='import-system-symbols', - short_help='Import system debug symbols.') -@click.argument('bundles', type=click.Path(), nargs=-1) -@click.option('--sdk', default='iOS', help='The SDK identifier') -@click.option('--dsym-type', default='macho', help='The type of the symbol') -@click.option('--threads', default=8, help='The number of threads to use') -@configuration -def import_system_symbols(bundles, sdk, dsym_type, threads): - """Imports system symbols from preprocessed zip files into Sentry. - - It takes a list of zip files as arguments that contain preprocessed - system symbol information. These zip files contain JSON dumps. The - actual zipped up dsym files cannot be used here, they need to be - preprocessed. - """ - import zipfile - for path in bundles: - with zipfile.ZipFile(path) as f: - sdk_info = json.load(f.open('sdk_info')) - sdk_info['sdk_name'] = sdk - sdk_info['dsym_type'] = dsym_type - label = ('%s.%s.%s (%s)' % ( - sdk_info['version_major'], - sdk_info['version_minor'], - sdk_info['version_patchlevel'], - sdk_info['version_build'], - )).ljust(18) - with click.progressbar(f.namelist(), label=label) as bar: - process_archive(bar, f, sdk_info, threads) From 7be386aaf290362327b791705638c060e39aeace Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 14 Apr 2016 15:30:34 +0200 Subject: [PATCH 11/17] Bump symsynd to 0.6.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df8d548e9cf6d9..8b4337b1ea9b46 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ ] dsym_requires = [ - 'symsynd>=0.6.0,<1.0.0', + 'symsynd>=0.6.1,<1.0.0', ] From 4ee5d7389ee818fa0a77ac62d13c8402d3bd70c1 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 14 Apr 2016 18:45:07 +0200 Subject: [PATCH 12/17] Added missing platforms to SDK mapping --- src/sentry/models/dsymfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index 34b6c1105337c3..d9b707018aa47a 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -33,6 +33,8 @@ SDK_MAPPING = { 'iPhone OS': 'iOS', + 'tvOS': 'tvOS', + 'Mac OS': 'macOS', } From 5e3884b32fa19bc20500b3d9c2175d3b38efa051 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 14 Apr 2016 18:54:08 +0200 Subject: [PATCH 13/17] Fix SENTRY-12T --- src/sentry/lang/native/dsymcache.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/sentry/lang/native/dsymcache.py b/src/sentry/lang/native/dsymcache.py index e5f0da26cf2e00..5e32d8fb4460d7 100644 --- a/src/sentry/lang/native/dsymcache.py +++ b/src/sentry/lang/native/dsymcache.py @@ -1,4 +1,5 @@ import os +import uuid import time import errno import shutil @@ -69,9 +70,21 @@ def fetch_dsym(self, project, image_uuid): pass with dsf.file.getfile() as sf: - with open(dsym + '_tmp', 'w') as df: - shutil.copyfileobj(sf, df) - os.rename(dsym + '_tmp', dsym) + suffix = '_%s' % uuid.uuid4() + done = False + try: + with open(dsym + suffix, 'w') as df: + shutil.copyfileobj(sf, df) + os.rename(dsym + suffix, dsym) + done = True + finally: + # Use finally here because it does not lie about the + # error on exit + if not done: + try: + os.remove(dsym + suffix) + except Exception: + pass return base, dsym From 027f110e9e510a57698264daf2ccb81cda9fa23f Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 19 Apr 2016 02:01:19 +0200 Subject: [PATCH 14/17] Stop using prepare for symbol filling --- src/sentry/models/dsymfile.py | 4 ++-- src/sentry/runner/commands/dsym.py | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index d9b707018aa47a..f17e53deb6109e 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -161,7 +161,7 @@ def lookup_symbol(self, instruction_addr, image_addr, uuid, try: # First try: exact match on uuid cur.execute(''' - select symbol + select s.symbol from sentry_dsymsymbol s, sentry_dsymobject o where o.uuid = %s and @@ -183,7 +183,7 @@ def lookup_symbol(self, instruction_addr, image_addr, uuid, return cur.execute(''' - select symbol + select s.symbol from sentry_dsymsymbol s, sentry_dsymobject o, sentry_dsymsdk k, diff --git a/src/sentry/runner/commands/dsym.py b/src/sentry/runner/commands/dsym.py index 8228ed0188ce8c..bf09dbe50d854e 100644 --- a/src/sentry/runner/commands/dsym.py +++ b/src/sentry/runner/commands/dsym.py @@ -80,19 +80,15 @@ def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False, def process_items(): cur = connection.cursor() cur.execute('begin') - cur.execute(''' - prepare add_sym(bigint, bigint, text) as - insert into sentry_dsymsymbol (object_id, address, symbol) - select $1, $2, $3 - where not exists (select 1 from sentry_dsymsymbol - where object_id = $1 and address = $2); - ''') while 1: items = q.get() if items is SHUTDOWN: break cur.executemany(''' - execute add_sym(%(object_id)s, %(address)s, %(symbol)s); + insert into sentry_dsymsymbol (object_id, address, symbol) + select %(object_id)s, %(address)s, %(symbol)s + where not exists (select 1 from sentry_dsymsymbol + where object_id = %(object_id)s and address = %(address)s); ''', items) cur.execute('commit') From 1b5dd37a53e7a4f6de40e41d14aed453f1d74cc7 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 19 Apr 2016 13:59:13 +0200 Subject: [PATCH 15/17] Added optimistic bulk insert with normal fallback --- src/sentry/runner/commands/dsym.py | 71 ++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/src/sentry/runner/commands/dsym.py b/src/sentry/runner/commands/dsym.py index bf09dbe50d854e..30f58509e8c685 100644 --- a/src/sentry/runner/commands/dsym.py +++ b/src/sentry/runner/commands/dsym.py @@ -11,8 +11,9 @@ import json import click import threading +from itertools import chain -from django.db import connection +from django.db import connection, transaction, IntegrityError from sentry.runner.decorators import configuration @@ -20,6 +21,10 @@ SHUTDOWN = object() +class Done(Exception): + pass + + def load_bundle(q, uuid, data, sdk_info, trim_symbols, demangle): from sentry.models import DSymBundle, DSymObject, DSymSDK from sentry.models.dsymfile import MAX_SYM @@ -61,15 +66,12 @@ def _process_symbol(sym): symbols = data['symbols'] for idx in xrange(0, len(symbols) + step, step): end_idx = min(idx + step, len(symbols)) - batch = {} + batch = [] for x in xrange(idx, end_idx): addr = symbols[x][0] - batch[obj.id, addr] = { - 'object_id': obj.id, - 'address': addr, - 'symbol': _process_symbol(symbols[x][1]), - } - yield sorted(batch.values(), key=lambda x: x['address']) + batch.append((obj.id, addr, _process_symbol(symbols[x][1]))) + if batch: + yield batch def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False, @@ -78,19 +80,50 @@ def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False, q = Queue.Queue(threads) def process_items(): - cur = connection.cursor() - cur.execute('begin') + items = None + can_bulk = True while 1: - items = q.get() - if items is SHUTDOWN: + try: + with transaction.atomic(): + cur = connection.cursor() + while 1: + if items is None: + items = q.get() + if items is SHUTDOWN: + raise Done + if not items: + continue + + if can_bulk: + bulk = ''' + insert into sentry_dsymsymbol + (object_id, address, symbol) + values %s + ''' % ', '.join(['(%s, %s, %s)'] * len(items)) + cur.execute(bulk, list(chain(*items))) + else: + for item in items: + cur.execute(''' + insert into sentry_dsymsymbol + (object_id, address, symbol) + select + %(object_id)s, %(address)s, %(symbol)s + where not exists ( + select 1 from sentry_dsymsymbol + where object_id = %(object_id)s + and address = %(address)s); + ''', { + 'object_id': item[0], + 'address': item[1], + 'symbol': item[2], + }) + items = None + can_bulk = True + except IntegrityError: + can_bulk = False + items = None + except Done: break - cur.executemany(''' - insert into sentry_dsymsymbol (object_id, address, symbol) - select %(object_id)s, %(address)s, %(symbol)s - where not exists (select 1 from sentry_dsymsymbol - where object_id = %(object_id)s and address = %(address)s); - ''', items) - cur.execute('commit') pool = [] for x in xrange(threads): From 033aaf1e3f932c7484c233f38aa1e16f74008a3b Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 19 Apr 2016 14:13:51 +0200 Subject: [PATCH 16/17] Lower dsym insert batch for sqlite --- src/sentry/runner/commands/dsym.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/sentry/runner/commands/dsym.py b/src/sentry/runner/commands/dsym.py index 30f58509e8c685..c61e3200ffee06 100644 --- a/src/sentry/runner/commands/dsym.py +++ b/src/sentry/runner/commands/dsym.py @@ -28,6 +28,7 @@ class Done(Exception): def load_bundle(q, uuid, data, sdk_info, trim_symbols, demangle): from sentry.models import DSymBundle, DSymObject, DSymSDK from sentry.models.dsymfile import MAX_SYM + from sentry.utils.db import is_sqlite from symsynd.demangle import demangle_symbol def _process_symbol(sym): @@ -62,7 +63,13 @@ def _process_symbol(sym): object=obj )[0] - step = 4000 + # SQlite has a low parameter limit of 999. Since we need three + # parameters to insert a row, we can only do 333 items in a batch + if is_sqlite(): + step = 333 + else: + step = 4000 + symbols = data['symbols'] for idx in xrange(0, len(symbols) + step, step): end_idx = min(idx + step, len(symbols)) @@ -121,7 +128,6 @@ def process_items(): can_bulk = True except IntegrityError: can_bulk = False - items = None except Done: break From cc4536901db0323d1e6433416abf1d0ecd977d61 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 19 Apr 2016 18:04:27 +0200 Subject: [PATCH 17/17] Move bulk insert code into the models --- src/sentry/models/dsymfile.py | 47 +++++++++++++++++++++++- src/sentry/runner/commands/dsym.py | 59 +++--------------------------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/sentry/models/dsymfile.py b/src/sentry/models/dsymfile.py index f17e53deb6109e..048e029dd9a43d 100644 --- a/src/sentry/models/dsymfile.py +++ b/src/sentry/models/dsymfile.py @@ -12,7 +12,8 @@ import shutil import hashlib import tempfile -from django.db import models, transaction, connection, IntegrityError +from itertools import chain +from django.db import models, router, transaction, connection, IntegrityError try: from symsynd.macho.arch import get_macho_uuids @@ -24,6 +25,7 @@ sane_repr, BaseManager from sentry.models.file import File from sentry.utils.zip import safe_extract_zip +from sentry.utils.db import is_sqlite MAX_SYM = 256 @@ -151,6 +153,49 @@ class Meta: class DSymSymbolManager(BaseManager): + def bulk_insert(self, items): + db = router.db_for_write(DSymSymbol) + items = list(items) + if not items: + return + + # On SQLite we don't do this. Two reasons: one, it does not + # seem significantly faster and you're an idiot if you import + # huge amounts of system symbols into sqlite anyways. secondly + # because of the low parameter limit + if not is_sqlite(): + try: + with transaction.atomic(using=db): + cur = connection.cursor() + cur.execute(''' + insert into sentry_dsymsymbol + (object_id, address, symbol) + values %s + ''' % ', '.join(['(%s, %s, %s)'] * len(items)), + list(chain(*items))) + cur.close() + return + except IntegrityError: + pass + + cur = connection.cursor() + for item in items: + cur.execute(''' + insert into sentry_dsymsymbol + (object_id, address, symbol) + select + %(object_id)s, %(address)s, %(symbol)s + where not exists ( + select 1 from sentry_dsymsymbol + where object_id = %(object_id)s + and address = %(address)s); + ''', { + 'object_id': item[0], + 'address': item[1], + 'symbol': item[2], + }) + cur.close() + def lookup_symbol(self, instruction_addr, image_addr, uuid, cpu_name=None, object_path=None, system_info=None): """Finds a system symbol.""" diff --git a/src/sentry/runner/commands/dsym.py b/src/sentry/runner/commands/dsym.py index c61e3200ffee06..fd28de8869bc11 100644 --- a/src/sentry/runner/commands/dsym.py +++ b/src/sentry/runner/commands/dsym.py @@ -11,9 +11,6 @@ import json import click import threading -from itertools import chain - -from django.db import connection, transaction, IntegrityError from sentry.runner.decorators import configuration @@ -28,7 +25,6 @@ class Done(Exception): def load_bundle(q, uuid, data, sdk_info, trim_symbols, demangle): from sentry.models import DSymBundle, DSymObject, DSymSDK from sentry.models.dsymfile import MAX_SYM - from sentry.utils.db import is_sqlite from symsynd.demangle import demangle_symbol def _process_symbol(sym): @@ -63,13 +59,7 @@ def _process_symbol(sym): object=obj )[0] - # SQlite has a low parameter limit of 999. Since we need three - # parameters to insert a row, we can only do 333 items in a batch - if is_sqlite(): - step = 333 - else: - step = 4000 - + step = 4000 symbols = data['symbols'] for idx in xrange(0, len(symbols) + step, step): end_idx = min(idx + step, len(symbols)) @@ -83,53 +73,16 @@ def _process_symbol(sym): def process_archive(members, zip, sdk_info, threads=8, trim_symbols=False, demangle=True): + from sentry.models import DSymSymbol import Queue q = Queue.Queue(threads) def process_items(): - items = None - can_bulk = True while 1: - try: - with transaction.atomic(): - cur = connection.cursor() - while 1: - if items is None: - items = q.get() - if items is SHUTDOWN: - raise Done - if not items: - continue - - if can_bulk: - bulk = ''' - insert into sentry_dsymsymbol - (object_id, address, symbol) - values %s - ''' % ', '.join(['(%s, %s, %s)'] * len(items)) - cur.execute(bulk, list(chain(*items))) - else: - for item in items: - cur.execute(''' - insert into sentry_dsymsymbol - (object_id, address, symbol) - select - %(object_id)s, %(address)s, %(symbol)s - where not exists ( - select 1 from sentry_dsymsymbol - where object_id = %(object_id)s - and address = %(address)s); - ''', { - 'object_id': item[0], - 'address': item[1], - 'symbol': item[2], - }) - items = None - can_bulk = True - except IntegrityError: - can_bulk = False - except Done: - break + items = q.get() + if items is SHUTDOWN: + raise Done + DSymSymbol.objects.bulk_insert(items) pool = [] for x in xrange(threads):