Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixes bug 1013321 - Admin UI for the master list of Super Search Fiel…

…ds. r=peterbe
  • Loading branch information...
commit 7960713def5a344fb259486c0f5c642301bb1402 1 parent 2c0e7a7
@AdrianGaudebert AdrianGaudebert authored
Showing with 1,958 additions and 71 deletions.
  1. 0  {socorro/external/elasticsearch → scripts/data}/supersearch_fields.json
  2. +100 −0 scripts/setup_supersearch_app.py
  3. +6 −0 socorro/external/elasticsearch/connection_context.py
  4. +185 −16 socorro/external/elasticsearch/supersearch.py
  5. +8 −0 socorro/lib/external_common.py
  6. +3 −0  socorro/lib/search_common.py
  7. +6 −0 socorro/middleware/middleware_app.py
  8. +14 −1 socorro/unittest/external/elasticsearch/test_settings.py
  9. +813 −7 socorro/unittest/external/elasticsearch/test_supersearch.py
  10. +1 −0  socorro/unittest/external/elasticsearch/unittestbase.py
  11. +1 −0  webapp-django/crashstats/api/views.py
  12. BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/application_edit.png
  13. BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/cross.png
  14. BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/tick.png
  15. +70 −43 webapp-django/crashstats/crashstats/models.py
  16. 0  ...ango/crashstats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/LICENSE
  17. 0  ...ats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/select2-spinner.gif
  18. 0  .../crashstats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/select2.css
  19. 0  ...o/crashstats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/select2.js
  20. 0  .../crashstats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/select2.png
  21. 0  ...rashstats/{supersearch/static/supersearch → crashstats/static/crashstats}/js/lib/select2/select2x2.png
  22. +19 −0 webapp-django/crashstats/manage/forms.py
  23. +48 −0 webapp-django/crashstats/manage/static/manage/css/supersearch_fields.less
  24. +63 −0 webapp-django/crashstats/manage/static/manage/js/supersearch_field.js
  25. +2 −0  webapp-django/crashstats/manage/templates/manage/base.html
  26. +164 −0 webapp-django/crashstats/manage/templates/manage/supersearch_field.html
  27. +93 −0 webapp-django/crashstats/manage/templates/manage/supersearch_fields.html
  28. +205 −0 webapp-django/crashstats/manage/tests/test_views.py
  29. +15 −0 webapp-django/crashstats/manage/urls.py
  30. +98 −0 webapp-django/crashstats/manage/views.py
  31. +36 −0 webapp-django/crashstats/supersearch/models.py
  32. +4 −2 webapp-django/crashstats/supersearch/templates/supersearch/search.html
  33. +4 −2 webapp-django/crashstats/supersearch/templates/supersearch/search_custom.html
View
0  ...ro/external/elasticsearch/supersearch_fields.json → scripts/data/supersearch_fields.json
File renamed without changes
View
100 scripts/setup_supersearch_app.py
@@ -0,0 +1,100 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Index supersearch fields data into elasticsearch.
+
+This script creates a first set of data to be used by Super Search as the list
+of fields it exposes to users, as well as to generate the elasticsearch
+mapping for processed and raw crashes.
+"""
+
+import json
+import os
+
+from configman import Namespace
+from configman.converters import class_converter
+
+from socorro.app import generic_app
+
+
+class SetupSuperSearchApp(generic_app.App):
+ """Index supersearch fields data into elasticsearch. """
+
+ app_name = 'setup-supersearch'
+ app_version = '1.0'
+ app_description = __doc__
+
+ required_config = Namespace()
+
+ required_config.add_option(
+ 'supersearch_fields_file',
+ default=os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ 'data',
+ 'supersearch_fields.json'
+ ),
+ )
+
+ required_config.namespace('elasticsearch')
+ required_config.elasticsearch.add_option(
+ 'elasticsearch_class',
+ default='socorro.external.elasticsearch.connection_context.'
+ 'ConnectionContext',
+ from_string_converter=class_converter,
+ )
+ required_config.elasticsearch.add_option(
+ 'index_creator_class',
+ default='socorro.external.elasticsearch.crashstorage.'
+ 'ElasticSearchCrashStorage',
+ from_string_converter=class_converter,
+ )
+
+ def main(self):
+ # Create the socorro index in elasticsearch.
+ index_creator = self.config.elasticsearch.index_creator_class(
+ self.config.elasticsearch
+ )
+ index_creator.create_index('socorro', None)
+
+ # Load the initial data set.
+ data_file = open(self.config.supersearch_fields_file, 'r')
+ all_fields = json.loads(data_file.read())
+
+ # Index the data.
+ es_connection = index_creator.es
+ es_connection.bulk_index(
+ index='socorro',
+ doc_type='supersearch_fields',
+ docs=all_fields.values(),
+ id_field='name',
+ )
+
+ # Verify data was correctly inserted.
+ es_connection.refresh()
+ total_indexed = es_connection.count(
+ '*',
+ index='socorro',
+ doc_type='supersearch_fields',
+ )['count']
+ total_expected = len(all_fields)
+
+ if total_expected != total_indexed:
+ indexed_fields = es_connection.search(
+ '*',
+ index='socorro',
+ doc_type='supersearch_fields',
+ size=total_indexed,
+ )
+ indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]
+
+ self.config.logger.error(
+ 'The SuperSearch fields data was not correctly indexed, '
+ '%s fields are missing from the database. Missing fields: %s',
+ total_expected - total_indexed,
+ list(set(all_fields.keys()) - set(indexed_fields))
+ )
+
+
+if __name__ == '__main__':
+ generic_app.main(SetupSuperSearchApp)
View
6 socorro/external/elasticsearch/connection_context.py
@@ -23,6 +23,12 @@ class ConnectionContext(RequiredConfig):
reference_value_from='resource.elasticsearch',
)
required_config.add_option(
+ 'elasticsearch_default_index',
+ default='socorro',
+ doc='the default index used to store data',
+ reference_value_from='resource.elasticsearch',
+ )
+ required_config.add_option(
'elasticsearch_index',
default='socorro%Y%W',
doc='an index format to pull crashes from elasticsearch '
View
201 socorro/external/elasticsearch/supersearch.py
@@ -2,16 +2,22 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import json
import re
-import os
from elasticutils import F, S
-from pyelasticsearch.exceptions import ElasticHttpNotFoundError
-
-from socorro.external import BadArgumentError
+from pyelasticsearch.exceptions import (
+ ElasticHttpError,
+ ElasticHttpNotFoundError,
+)
+
+from socorro.external import (
+ BadArgumentError,
+ InsertionError,
+ MissingArgumentError,
+ ResourceNotFound,
+)
from socorro.external.elasticsearch.base import ElasticSearchBase
-from socorro.lib import datetimeutil
+from socorro.lib import datetimeutil, external_common
from socorro.lib.search_common import SearchBase
@@ -34,8 +40,16 @@ def process_filter_missing(self, key, value, action):
class SuperSearch(SearchBase, ElasticSearchBase):
+ # Defining some filters for the field service that need to be considered
+ # as lists.
+ filters = [
+ ('form_field_choices', None, ['list', 'str']),
+ ('permissions_needed', None, ['list', 'str']),
+ ]
+
def __init__(self, *args, **kwargs):
config = kwargs.get('config')
+ ElasticSearchBase.__init__(self, config=config)
self.all_fields = self.get_fields()
@@ -48,13 +62,18 @@ def __init__(self, *args, **kwargs):
# init is mandatory.
# See http://freshfoo.com/blog/object__init__takes_no_parameters
SearchBase.__init__(self, config=config, fields=self.all_fields)
- ElasticSearchBase.__init__(self, config=config)
+
+ def get_connection(self):
+ return SuperS().es(
+ urls=self.config.elasticsearch_urls,
+ timeout=self.config.elasticsearch_timeout,
+ )
def get(self, **kwargs):
"""Return a list of results and facets based on parameters.
The list of accepted parameters (with types and default values) is in
- socorro.lib.search_common.SearchBase
+ the database and can be accessed with the supersearch_fields service.
"""
# Filter parameters and raise potential errors.
params = self.get_parameters(**kwargs)
@@ -63,10 +82,7 @@ def get(self, **kwargs):
indexes = self.get_indexes(params['date'])
# Create and configure the search object.
- search = SuperS().es(
- urls=self.config.elasticsearch_urls,
- timeout=self.config.elasticsearch_timeout,
- )
+ search = self.get_connection()
search = search.indexes(*indexes)
search = search.doctypes(self.config.elasticsearch_doctype)
@@ -304,8 +320,161 @@ def format_field_names(self, hit):
return new_hit
def get_fields(self):
- file_path = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'supersearch_fields.json'
+ """ Return all the fields from our database, as a dict where field
+ names are the keys.
+
+ No parameters are accepted.
+ """
+ # Create and configure the search object.
+ search = self.get_connection()
+ search = search.indexes(
+ self.config.elasticsearch_default_index
+ )
+ search = search.doctypes('supersearch_fields')
+
+ count = search.count() # Total number of results.
+ search = search[:count]
+
+ # Get all fields from the database.
+ return dict((r['name'], r) for r in search.values_dict())
+
+ def create_field(self, **kwargs):
+ """Create a new field in the database, to be used by supersearch and
+ all elasticsearch related services.
+ """
+ filters = [
+ ('name', None, 'str'),
+ ('data_validation_type', 'enum', 'str'),
+ ('default_value', None, 'str'),
+ ('description', None, 'str'),
+ ('form_field_type', 'MultipleValueField', 'str'),
+ ('form_field_choices', None, ['list', 'str']),
+ ('has_full_version', False, 'bool'),
+ ('in_database_name', None, 'str'),
+ ('is_exposed', False, 'bool'),
+ ('is_returned', False, 'bool'),
+ ('is_mandatory', False, 'bool'),
+ ('query_type', 'enum', 'str'),
+ ('namespace', None, 'str'),
+ ('permissions_needed', None, ['list', 'str']),
+ ('storage_mapping', None, 'json'),
+ ]
+ params = external_common.parse_arguments(filters, kwargs)
+
+ mandatory_params = ('name', 'in_database_name')
+ for param in mandatory_params:
+ if not params[param]:
+ raise MissingArgumentError(param)
+
+ es_connection = self.get_connection().get_es()
+
+ try:
+ es_connection.index(
+ index=self.config.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ doc=params,
+ id=params['name'],
+ overwrite_existing=False,
+ refresh=True,
+ )
+ except ElasticHttpError, e:
+ if e.status_code == 409:
+ # This field exists in the database, it thus cannot be created!
+ raise InsertionError(
+ 'The field "%s" already exists in the database, '
+ 'impossible to create it. ' % params['name']
+ )
+
+ # Else this is an unexpected error and we want to know about it.
+ raise
+
+ return True
+
+ def update_field(self, **kwargs):
+ """Update an existing field in the database.
+
+ If the field does not exist yet, a ResourceNotFound error is raised.
+
+ If you want to update only some keys, just do not pass the ones you
+ don't want to change.
+ """
+ filters = [
+ ('name', None, 'str'),
+ ('data_validation_type', None, 'str'),
+ ('default_value', None, 'str'),
+ ('description', None, 'str'),
+ ('form_field_type', None, 'str'),
+ ('form_field_choices', None, ['list', 'str']),
+ ('has_full_version', None, 'bool'),
+ ('in_database_name', None, 'str'),
+ ('is_exposed', None, 'bool'),
+ ('is_returned', None, 'bool'),
+ ('is_mandatory', None, 'bool'),
+ ('query_type', None, 'str'),
+ ('namespace', None, 'str'),
+ ('permissions_needed', None, ['list', 'str']),
+ ('storage_mapping', None, 'json'),
+ ]
+ params = external_common.parse_arguments(filters, kwargs)
+
+ if not params['name']:
+ raise MissingArgumentError('name')
+
+ # Remove all the parameters that were not explicitely passed.
+ for key in params.keys():
+ if key not in kwargs:
+ del params[key]
+
+ es_connection = self.get_connection().get_es()
+
+ # First verify that the field does exist.
+ try:
+ es_connection.get(
+ index=self.config.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id=params['name'],
+ )
+ except ElasticHttpNotFoundError:
+ # This field does not exist yet, it thus cannot be updated!
+ raise ResourceNotFound(
+ 'The field "%s" does not exist in the database, it needs to '
+ 'be created before it can be updated. ' % params['name']
+ )
+
+ # Then update the new field in the database. Note that pyelasticsearch
+ # takes care of merging the new document into the old one, so missing
+ # values won't be changed.
+ es_connection.update(
+ index=self.config.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ doc=params,
+ id=params['name'],
+ refresh=True,
+ )
+
+ return True
+
+ def delete_field(self, **kwargs):
+ """Remove a field from the database.
+
+ Removing a field means that it won't be indexed in elasticsearch
+ anymore, nor will it be exposed or accessible via supersearch. It
+ doesn't delete the data from crash reports though, so it would be
+ possible to re-create the field and reindex some indices to get that
+ data back.
+ """
+ filters = [
+ ('name', None, 'str'),
+ ]
+ params = external_common.parse_arguments(filters, kwargs)
+
+ if not params['name']:
+ raise MissingArgumentError('name')
+
+ es_connection = self.get_connection().get_es()
+ es_connection.delete(
+ index=self.config.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id=params['name'],
+ refresh=True,
)
- return json.loads(open(file_path, 'r').read())
View
8 socorro/lib/external_common.py
@@ -6,6 +6,8 @@
Common functions for external modules.
"""
+import json
+
from datetime import datetime, timedelta, date
from socorro.lib.util import DotDict
@@ -117,4 +119,10 @@ def check_type(param, datatype):
except ValueError:
param = None
+ elif datatype == "json" and isinstance(param, basestring):
+ try:
+ param = json.loads(param)
+ except ValueError:
+ param = None
+
return param
View
3  socorro/lib/search_common.py
@@ -7,6 +7,7 @@
"""
import datetime
+import json
import socorro.lib.external_common as extern
from socorro.external import BadArgumentError, MissingArgumentError
@@ -328,6 +329,8 @@ def convert_to_type(value, data_type):
value = datetimeutil.string_to_datetime(value)
elif data_type == 'date' and not isinstance(value, datetime.date):
value = datetimeutil.string_to_datetime(value).date()
+ elif data_type == 'json' and isinstance(value, basestring):
+ value = json.loads(value)
return value
View
6 socorro/middleware/middleware_app.py
@@ -226,6 +226,12 @@ class MiddlewareApp(App):
from_string_converter=string_to_list,
reference_value_from='resource.elasticsearch',
)
+ required_config.add_option(
+ 'elasticsearch_default_index',
+ default='socorro',
+ doc='the default index used to store data',
+ reference_value_from='resource.elasticsearch',
+ )
required_config.webapi.add_option(
'elasticsearch_index',
default='socorro%Y%W',
View
15 socorro/unittest/external/elasticsearch/test_settings.py
@@ -10,6 +10,9 @@
from socorro.external.elasticsearch.supersearch import SuperSearch
from socorro.lib.datetimeutil import utc_now
from .unittestbase import ElasticSearchTestCase
+from .test_supersearch import (
+ SUPERSEARCH_FIELDS
+)
# Remove debugging noise during development
# import logging
@@ -36,7 +39,6 @@ def setUp(self):
config = self.get_config_context()
self.storage = crashstorage.ElasticSearchCrashStorage(config)
- self.api = SuperSearch(config=config)
# clear the indices cache so the index is created on every test
self.storage.indices_cache = set()
@@ -47,6 +49,17 @@ def setUp(self):
es_index = self.storage.get_index_for_crash(self.now)
self.storage.create_socorro_index(es_index)
+ # Create the supersearch fields.
+ self.storage.es.bulk_index(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ docs=SUPERSEARCH_FIELDS.values(),
+ id_field='name',
+ refresh=True,
+ )
+
+ self.api = SuperSearch(config=config)
+
# This an ugly hack to give elasticsearch some time to finish creating
# the new index. It is needed for jenkins only, because we have a
# special case here where we index only one or two documents before
View
820 socorro/unittest/external/elasticsearch/test_supersearch.py
@@ -4,21 +4,666 @@
import datetime
import mock
+import pyelasticsearch
+
from nose.plugins.attrib import attr
from nose.tools import eq_, ok_, assert_raises
-from socorro.external import BadArgumentError
+from socorro.external import (
+ BadArgumentError,
+ InsertionError,
+ MissingArgumentError,
+ ResourceNotFound,
+)
from socorro.external.elasticsearch import crashstorage
from socorro.external.elasticsearch.supersearch import SuperSearch
from socorro.lib import datetimeutil, search_common
from .unittestbase import ElasticSearchTestCase
# Remove debugging noise during development
-# import logging
-# logging.getLogger('pyelasticsearch').setLevel(logging.ERROR)
-# logging.getLogger('elasticutils').setLevel(logging.ERROR)
-# logging.getLogger('requests.packages.urllib3.connectionpool')\
-# .setLevel(logging.ERROR)
+import logging
+logging.getLogger('pyelasticsearch').setLevel(logging.ERROR)
+logging.getLogger('elasticutils').setLevel(logging.ERROR)
+logging.getLogger('requests.packages.urllib3.connectionpool')\
+ .setLevel(logging.ERROR)
+
+
+SUPERSEARCH_FIELDS = {
+ 'signature': {
+ 'name': 'signature',
+ 'in_database_name': 'signature',
+ 'data_validation_type': 'str',
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': True,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'product': {
+ 'name': 'product',
+ 'in_database_name': 'product',
+ 'data_validation_type': 'enum',
+ 'query_type': 'enum',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': True,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'version': {
+ 'name': 'version',
+ 'in_database_name': 'version',
+ 'data_validation_type': 'enum',
+ 'query_type': 'enum',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'platform': {
+ 'name': 'platform',
+ 'in_database_name': 'os_name',
+ 'data_validation_type': 'enum',
+ 'query_type': 'enum',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': True,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'release_channel': {
+ 'name': 'release_channel',
+ 'in_database_name': 'release_channel',
+ 'data_validation_type': 'enum',
+ 'query_type': 'enum',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'date': {
+ 'name': 'date',
+ 'in_database_name': 'date_processed',
+ 'data_validation_type': 'datetime',
+ 'query_type': 'date',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'DateTimeField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'address': {
+ 'name': 'address',
+ 'in_database_name': 'address',
+ 'data_validation_type': 'str',
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'build_id': {
+ 'name': 'build_id',
+ 'in_database_name': 'build',
+ 'data_validation_type': 'int',
+ 'query_type': 'number',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'IntegerField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'reason': {
+ 'name': 'reason',
+ 'in_database_name': 'reason',
+ 'data_validation_type': 'str',
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'email': {
+ 'name': 'email',
+ 'in_database_name': 'email',
+ 'data_validation_type': 'str',
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': ['crashstats.view_pii'],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'url': {
+ 'name': 'url',
+ 'in_database_name': 'url',
+ 'data_validation_type': 'str',
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': ['crashstats.view_pii'],
+ 'default_value': None,
+ 'has_full_version': False,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'uuid': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': None,
+ 'has_full_version': False,
+ 'in_database_name': 'uuid',
+ 'is_exposed': False,
+ 'is_mandatory': False,
+ 'is_returned': False,
+ 'name': 'uuid',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'process_type': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': [
+ 'any', 'browser', 'plugin', 'content', 'all'
+ ],
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'process_type',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'process_type',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'type': 'string'
+ }
+ },
+ 'user_comments': {
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'StringField',
+ 'has_full_version': True,
+ 'in_database_name': 'user_comments',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'user_comments',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'string',
+ 'storage_mapping': {
+ 'fields': {
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ },
+ 'user_comments': {
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'accessibility': {
+ 'data_validation_type': 'bool',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'BooleanField',
+ 'has_full_version': False,
+ 'in_database_name': 'Accessibility',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'accessibility',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'bool',
+ 'storage_mapping': {
+ 'type': 'boolean'
+ }
+ },
+ 'b2g_os_version': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'B2G_OS_Version',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'b2g_os_version',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'analyzer': 'keyword',
+ 'type': 'string'
+ }
+ },
+ 'bios_manufacturer': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'BIOS_Manufacturer',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'bios_manufacturer',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'analyzer': 'keyword',
+ 'type': 'string'
+ }
+ },
+ 'vendor': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'Vendor',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'vendor',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'type': 'string'
+ }
+ },
+ 'useragent_locale': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'useragent_locale',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'useragent_locale',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'analyzer': 'keyword',
+ 'type': 'string'
+ }
+ },
+ 'is_garbage_collecting': {
+ 'data_validation_type': 'bool',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'BooleanField',
+ 'has_full_version': False,
+ 'in_database_name': 'IsGarbageCollecting',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'is_garbage_collecting',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'bool',
+ 'storage_mapping': {
+ 'type': 'boolean'
+ }
+ },
+ 'available_virtual_memory': {
+ 'data_validation_type': 'int',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'IntegerField',
+ 'has_full_version': False,
+ 'in_database_name': 'AvailableVirtualMemory',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'available_virtual_memory',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'number',
+ 'storage_mapping': {
+ 'type': 'long'
+ }
+ },
+ 'install_age': {
+ 'data_validation_type': 'int',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'IntegerField',
+ 'has_full_version': False,
+ 'in_database_name': 'install_age',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'install_age',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'number',
+ 'storage_mapping': {
+ 'type': 'long'
+ }
+ },
+ 'plugin_filename': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': True,
+ 'in_database_name': 'PluginFilename',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'plugin_filename',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'fields': {
+ 'PluginFilename': {
+ 'index': 'analyzed',
+ 'type': 'string'
+ },
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'plugin_name': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': True,
+ 'in_database_name': 'PluginName',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'plugin_name',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'fields': {
+ 'PluginName': {
+ 'index': 'analyzed',
+ 'type': 'string'
+ },
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'plugin_version': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': True,
+ 'in_database_name': 'PluginVersion',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'plugin_version',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'fields': {
+ 'PluginVersion': {
+ 'index': 'analyzed',
+ 'type': 'string'
+ },
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'android_model': {
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'StringField',
+ 'has_full_version': True,
+ 'in_database_name': 'Android_Model',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'android_model',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'string',
+ 'storage_mapping': {
+ 'fields': {
+ 'Android_Model': {
+ 'type': 'string'
+ },
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'dump': {
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'StringField',
+ 'has_full_version': False,
+ 'in_database_name': 'dump',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'dump',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'string',
+ 'storage_mapping': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'cpu_info': {
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'StringField',
+ 'has_full_version': True,
+ 'in_database_name': 'cpu_info',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'cpu_info',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'string',
+ 'storage_mapping': {
+ 'fields': {
+ 'cpu_info': {
+ 'analyzer': 'standard',
+ 'index': 'analyzed',
+ 'type': 'string'
+ },
+ 'full': {
+ 'index': 'not_analyzed',
+ 'type': 'string'
+ }
+ },
+ 'type': 'multi_field'
+ }
+ },
+ 'dom_ipc_enabled': {
+ 'data_validation_type': 'bool',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'BooleanField',
+ 'has_full_version': False,
+ 'in_database_name': 'DOMIPCEnabled',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'dom_ipc_enabled',
+ 'namespace': 'raw_crash',
+ 'permissions_needed': [],
+ 'query_type': 'bool',
+ 'storage_mapping': {
+ 'None_value': False,
+ 'type': 'boolean'
+ }
+ },
+ 'app_notes': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'app_notes',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'app_notes',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'type': 'string'
+ }
+ },
+ 'hang_type': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': [
+ 'any', 'crash', 'hang', 'all'
+ ],
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'hang_type',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'hang_type',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'type': 'short'
+ }
+ },
+ 'exploitability': {
+ 'data_validation_type': 'enum',
+ 'default_value': None,
+ 'form_field_choices': [
+ 'high', 'normal', 'low', 'none', 'unknown', 'error'
+ ],
+ 'form_field_type': 'MultipleValueField',
+ 'has_full_version': False,
+ 'in_database_name': 'exploitability',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'exploitability',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [
+ 'crashstats.view_exploitability'
+ ],
+ 'query_type': 'enum',
+ 'storage_mapping': {
+ 'type': 'string'
+ }
+ },
+ 'platform_version': {
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'form_field_choices': None,
+ 'form_field_type': 'StringField',
+ 'has_full_version': False,
+ 'in_database_name': 'os_version',
+ 'is_exposed': True,
+ 'is_mandatory': False,
+ 'is_returned': True,
+ 'name': 'platform_version',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': [],
+ 'query_type': 'string',
+ 'storage_mapping': {
+ 'type': 'string'
+ }
+ },
+}
class TestSuperSearch(ElasticSearchTestCase):
@@ -26,6 +671,17 @@ class TestSuperSearch(ElasticSearchTestCase):
def test_get_indexes(self):
config = self.get_config_context()
+ storage = crashstorage.ElasticSearchCrashStorage(config)
+
+ # Create the supersearch fields.
+ storage.es.bulk_index(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ docs=SUPERSEARCH_FIELDS.values(),
+ id_field='name',
+ refresh=True,
+ )
+
api = SuperSearch(config=config)
now = datetime.datetime(2000, 2, 1, 0, 0)
@@ -75,7 +731,6 @@ def setUp(self):
super(IntegrationTestSuperSearch, self).setUp()
config = self.get_config_context()
- self.api = SuperSearch(config=config)
self.storage = crashstorage.ElasticSearchCrashStorage(config)
# clear the indices cache so the index is created on every test
@@ -263,10 +918,21 @@ def setUp(self):
dict(default_crash_report, uuid=21, address='0xa2e4509ca0')
)
+ # Create the supersearch fields.
+ self.storage.es.bulk_index(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ docs=SUPERSEARCH_FIELDS.values(),
+ id_field='name',
+ refresh=True,
+ )
+
# As indexing is asynchronous, we need to force elasticsearch to
# make the newly created content searchable before we run the tests
self.storage.es.refresh()
+ self.api = SuperSearch(config=config)
+
def tearDown(self):
# clear the test index
config = self.get_config_context()
@@ -896,3 +1562,143 @@ def test_return_query_mode(self):
ok_('filter' in query)
ok_('facets' in query)
ok_('size' in query)
+
+ def test_create_field(self):
+ es = self.storage.es
+ config = self.get_config_context()
+
+ # Test with all parameters set.
+ params = {
+ 'name': 'plotfarm',
+ 'data_validation_type': 'str',
+ 'default_value': None,
+ 'description': 'a plotfarm like Lunix or Wondiws',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': ['lun', 'won', 'cam'],
+ 'has_full_version': True,
+ 'in_database_name': 'os_name',
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ 'query_type': 'str',
+ 'namespace': 'processed_crash',
+ 'permissions_needed': ['view_plotfarm'],
+ 'storage_mapping': {"type": "multi_field"},
+ }
+ res = self.api.create_field(**params)
+ ok_(res)
+ field = es.get(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id='plotfarm',
+ )
+ field = field['_source']
+ eq_(sorted(field.keys()), sorted(params.keys()))
+ for key in field.keys():
+ eq_(field[key], params[key])
+
+ # Test default values.
+ res = self.api.create_field(
+ name='brand_new_field',
+ in_database_name='brand_new_field',
+ namespace='processed_crash',
+ )
+ ok_(res)
+ ok_(
+ es.get(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id='brand_new_field',
+ )
+ )
+
+ # Test errors.
+ assert_raises(
+ MissingArgumentError,
+ self.api.create_field,
+ in_database_name='something',
+ ) # `name` is missing
+ assert_raises(
+ MissingArgumentError,
+ self.api.create_field,
+ name='something',
+ ) # `in_database_name` is missing
+
+ assert_raises(
+ InsertionError,
+ self.api.create_field,
+ name='product',
+ in_database_name='product',
+ namespace='processed_crash',
+ )
+
+ def test_update_field(self):
+ es = self.storage.es
+ config = self.get_config_context()
+
+ # Let's create a field first.
+ assert self.api.create_field(
+ name='super_field',
+ in_database_name='super_field',
+ namespace='superspace',
+ description='inaccurate description',
+ permissions_needed=['view_nothing'],
+ storage_mapping='{"type": "boolean"}'
+ )
+
+ # Now let's update that field a little.
+ res = self.api.update_field(
+ name='super_field',
+ description='very accurate description',
+ storage_mapping={'type': 'long', 'analyzer': 'custom'},
+ )
+ ok_(res)
+
+ field = es.get(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id='super_field',
+ )
+ field = field['_source']
+
+ # Verify the changes were taken into account.
+ eq_(field['description'], 'very accurate description')
+ eq_(field['storage_mapping'], {'type': 'long', 'analyzer': 'custom'})
+
+ # Verify other values did not change.
+ eq_(field['permissions_needed'], ['view_nothing'])
+ eq_(field['in_database_name'], 'super_field')
+ eq_(field['namespace'], 'superspace')
+
+ # Test errors.
+ assert_raises(
+ MissingArgumentError,
+ self.api.update_field,
+ ) # `name` is missing
+
+ assert_raises(
+ ResourceNotFound,
+ self.api.update_field,
+ name='unkownfield',
+ )
+
+ def test_delete_field(self):
+ es = self.storage.es
+ config = self.get_config_context()
+
+ self.api.delete_field(name='product')
+
+ ok_(
+ es.get(
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id='signature',
+ )
+ )
+ assert_raises(
+ pyelasticsearch.exceptions.ElasticHttpNotFoundError,
+ es.get,
+ index=config.webapi.elasticsearch_default_index,
+ doc_type='supersearch_fields',
+ id='product',
+ )
View
1  socorro/unittest/external/elasticsearch/unittestbase.py
@@ -25,6 +25,7 @@ def get_config_context(self, es_index=None):
values_source = {
'logger': mock_logging,
+ 'elasticsearch_default_index': 'socorro_integration_test',
'elasticsearch_index': 'socorro_integration_test',
'backoff_delays': [1],
'elasticsearch_timeout': 5,
View
1  webapp-django/crashstats/api/views.py
@@ -148,6 +148,7 @@ class FormWrapper(forms.Form):
'ReleasesFeatured',
# because it's only used for the admin
'Field',
+ 'SuperSearchField',
# because it's very sensitive and we don't want to expose it
'Query',
# because it's an internal thing only
View
BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/application_edit.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/cross.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN  webapp-django/crashstats/base/static/img/3rdparty/silk/tick.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
113 webapp-django/crashstats/crashstats/models.py
@@ -118,11 +118,19 @@ class SocorroCommon(object):
cache_seconds = 60 * 60
@measure_fetches
- def fetch(self, url, headers=None, method='get', params=None, data=None,
- expect_json=True, dont_cache=False,
- retries=None,
- retry_sleeptime=None):
-
+ def fetch(
+ self,
+ url,
+ headers=None,
+ method='get',
+ params=None,
+ data=None,
+ expect_json=True,
+ dont_cache=False,
+ refresh_cache=False,
+ retries=None,
+ retry_sleeptime=None
+ ):
if retries is None:
retries = settings.MIDDLEWARE_RETRIES
if retry_sleeptime is None:
@@ -156,47 +164,58 @@ def fetch(self, url, headers=None, method='get', params=None, data=None,
data=data,
params=params,
).prepare()
-
cache_key = md5_constructor(iri_to_uri(req.url)).hexdigest()
- result = cache.get(cache_key)
- if result is not None:
- logger.debug("CACHE HIT %s" % url)
- return result, True
-
- # not in the memcache/locmem but is it in cache files?
- if settings.CACHE_MIDDLEWARE_FILES:
- root = settings.CACHE_MIDDLEWARE_FILES
- if isinstance(root, bool):
- cache_file = os.path.join(settings.ROOT, 'models-cache')
- else:
- cache_file = root
- split = urlparse.urlparse(url)
- cache_file = os.path.join(cache_file,
- split.netloc,
- _clean_path(split.path))
- if split.query:
- cache_file = os.path.join(cache_file,
- _clean_query(split.query))
- if expect_json:
- cache_file = os.path.join(cache_file,
- '%s.json' % cache_key)
- else:
- cache_file = os.path.join(cache_file,
- '%s.dump' % cache_key)
-
- if os.path.isfile(cache_file):
- # but is it fresh enough?
- age = time.time() - os.stat(cache_file)[stat.ST_MTIME]
- if age > self.cache_seconds:
- logger.debug("CACHE FILE TOO OLD")
- os.remove(cache_file)
+ if not refresh_cache:
+ result = cache.get(cache_key)
+ if result is not None:
+ logger.debug("CACHE HIT %s" % url)
+ return result, True
+
+ # not in the memcache/locmem but is it in cache files?
+ if settings.CACHE_MIDDLEWARE_FILES:
+ root = settings.CACHE_MIDDLEWARE_FILES
+ if isinstance(root, bool):
+ cache_file = os.path.join(
+ settings.ROOT,
+ 'models-cache'
+ )
+ else:
+ cache_file = root
+ split = urlparse.urlparse(url)
+ cache_file = os.path.join(
+ cache_file,
+ split.netloc,
+ _clean_path(split.path)
+ )
+ if split.query:
+ cache_file = os.path.join(
+ cache_file,
+ _clean_query(split.query)
+ )
+ if expect_json:
+ cache_file = os.path.join(
+ cache_file,
+ '%s.json' % cache_key
+ )
else:
- logger.debug("CACHE FILE HIT %s" % url)
- if expect_json:
- return json.load(open(cache_file)), True
+ cache_file = os.path.join(
+ cache_file,
+ '%s.dump' % cache_key
+ )
+
+ if os.path.isfile(cache_file):
+ # but is it fresh enough?
+ age = time.time() - os.stat(cache_file)[stat.ST_MTIME]
+ if age > self.cache_seconds:
+ logger.debug("CACHE FILE TOO OLD")
+ os.remove(cache_file)
else:
- return open(cache_file).read(), True
+ logger.debug("CACHE FILE HIT %s" % url)
+ if expect_json:
+ return json.load(open(cache_file)), True
+ else:
+ return open(cache_file).read(), True
if method == 'post':
request_method = requests.post
@@ -306,7 +325,14 @@ def _post(self, url, payload, method='post'):
dont_cache=True,
)
- def _get(self, method='get', dont_cache=False, expect_json=True, **kwargs):
+ def _get(
+ self,
+ method='get',
+ dont_cache=False,
+ refresh_cache=False,
+ expect_json=True,
+ **kwargs
+ ):
"""
This is the generic `get` method that will take
`self.required_params` and `self.possible_params` and construct
@@ -336,6 +362,7 @@ def _get(self, method='get', dont_cache=False, expect_json=True, **kwargs):
params=params,
method=method,
dont_cache=dont_cache,
+ refresh_cache=refresh_cache,
expect_json=expect_json,
)
View
0  ...rsearch/static/supersearch/js/lib/select2/LICENSE → ...ashstats/static/crashstats/js/lib/select2/LICENSE
File renamed without changes
View
0  ...ic/supersearch/js/lib/select2/select2-spinner.gif → ...tic/crashstats/js/lib/select2/select2-spinner.gif
File renamed without changes
View
0  ...rch/static/supersearch/js/lib/select2/select2.css → ...tats/static/crashstats/js/lib/select2/select2.css
File renamed without changes
View
0  ...arch/static/supersearch/js/lib/select2/select2.js → ...stats/static/crashstats/js/lib/select2/select2.js
File renamed without changes
View
0  ...rch/static/supersearch/js/lib/select2/select2.png → ...tats/static/crashstats/js/lib/select2/select2.png
File renamed without changes
View
0  ...h/static/supersearch/js/lib/select2/select2x2.png → ...ts/static/crashstats/js/lib/select2/select2x2.png
File renamed without changes
View
19 webapp-django/crashstats/manage/forms.py
@@ -2,6 +2,7 @@
from django import forms
from crashstats.crashstats.forms import BaseForm, BaseModelForm
+from crashstats.supersearch import form_fields
class SkipListForm(BaseForm):
@@ -69,3 +70,21 @@ class GraphicsDeviceLookupForm(BaseForm):
class GraphicsDeviceUploadForm(BaseForm):
file = forms.FileField()
+
+
+class SuperSearchFieldForm(BaseForm):
+
+ name = forms.CharField()
+ in_database_name = forms.CharField()
+ namespace = forms.CharField(required=False)
+ description = forms.CharField(required=False)
+ query_type = forms.CharField(required=False)
+ data_validation_type = forms.CharField(required=False)
+ permissions_needed = form_fields.MultipleValueField(required=False)
+ form_field_type = forms.CharField(required=False)
+ form_field_choices = form_fields.MultipleValueField(required=False)
+ is_exposed = forms.BooleanField(required=False)
+ is_returned = forms.BooleanField(required=False)
+ is_mandatory = forms.BooleanField(required=False)
+ has_full_version = forms.BooleanField(required=False)
+ storage_mapping = forms.CharField(required=False)
View
48 webapp-django/crashstats/manage/static/manage/css/supersearch_fields.less
@@ -0,0 +1,48 @@
+
+a.modify,
+a.delete,
+span.false,
+span.true {
+ background: url('../../img/3rdparty/silk/tick.png') no-repeat;
+ display: inline-block;
+ height: 16px;
+ width: 16px;
+ text-indent: -9999px;
+}
+span.false {
+ background: url('../../img/3rdparty/silk/cross.png') no-repeat;
+}
+
+a.modify {
+ background: url('../../img/3rdparty/silk/application_edit.png') no-repeat;
+}
+a.delete {
+ background: url('../../img/3rdparty/silk/delete.png') no-repeat;
+}
+
+td.boolean {
+ text-align: center;
+ vertical-align: middle;
+}
+
+#supersearch-field input[type=text],
+#supersearch-field select,
+#supersearch-field textarea {
+ width: 500px;
+}
+#supersearch-field textarea {
+ height: 150px;
+}
+
+.create-field {
+ float: right;
+ margin-left: 20px;
+}
+
+.create-field:before {
+ content: url('../../img/3rdparty/silk/application_form_add.png');
+ height: 16px;
+ margin: 4px;
+ vertical-align: middle;
+ width: 16px;
+}
View
63 webapp-django/crashstats/manage/static/manage/js/supersearch_field.js
@@ -0,0 +1,63 @@
+/*global: ALL_PERMISSIONS */
+
+$(function () {
+ 'use strict';
+
+ var formElt = $('#supersearch-field');
+
+ $('input[name=name]', formElt).select2({
+ 'tags': [],
+ 'maximumSelectionSize': 1,
+ 'width': 'element'
+ });
+
+ $('input[name=namespace]', formElt).select2({
+ 'tags': [
+ 'processed_crash',
+ 'raw_crash'
+ ],
+ 'maximumSelectionSize': 1,
+ 'width': 'element'
+ });
+
+ $('input[name=in_database_name]', formElt).select2({
+ 'tags': [],
+ 'maximumSelectionSize': 1,
+ 'width': 'element'
+ });
+
+ var queryTypeElt = $('select[name=query_type]', formElt);
+ queryTypeElt.select2({
+ 'width': 'element'
+ });
+ if (queryTypeElt.data('selected')) {
+ queryTypeElt.select2('val', queryTypeElt.data('selected'));
+ }
+
+ var dataValidationTypeElt = $('select[name=data_validation_type]', formElt);
+ dataValidationTypeElt.select2({
+ 'width': 'element'
+ });
+ if (dataValidationTypeElt.data('selected')) {
+ dataValidationTypeElt.select2('val', dataValidationTypeElt.data('selected'));
+ }
+
+ $('input[name=permissions_needed]', formElt).select2({
+ 'tags': ALL_PERMISSIONS,
+ 'width': 'element'
+ });
+
+ var formFieldTypeElt = $('select[name=form_field_type]', formElt);
+ formFieldTypeElt.select2({
+ 'width': 'element'
+ });
+ if (formFieldTypeElt.data('selected')) {
+ formFieldTypeElt.select2('val', formFieldTypeElt.data('selected'));
+ }
+
+ $('input[name=form_field_choices]', formElt).select2({
+ 'tags': [],
+ 'width': 'element'
+ });
+
+});
View
2  webapp-django/crashstats/manage/templates/manage/base.html
@@ -27,6 +27,8 @@
{% if url('manage:graphics_devices') in request.path_info %}class="selected"{% endif %}>Graphics Devices</a></li>
<li><a href="{{ url('manage:symbols_uploads') }}"
{% if url('manage:symbols_uploads') in request.path_info %}class="selected"{% endif %}>Symbols Uploads</a></li>
+ <li><a href="{{ url('manage:supersearch_fields') }}"
+ {% if url('manage:supersearch_fields') in request.path_info %}class="selected"{% endif %}>Super Search Fields</a></li>
</ul>
</div>
{% block mainbody %}
View
164 webapp-django/crashstats/manage/templates/manage/supersearch_field.html
@@ -0,0 +1,164 @@
+{% extends "manage/base.html" %}
+
+{% block page_title %}{{ super() }} - Super Search Fields{% endblock %}
+
+{% block admin_title %}{{ super() }} - Super Search Fields{% endblock %}
+
+{% block site_js %}
+ {{ super() }}
+
+ {% compress js %}
+<script src="{{ static('crashstats/js/lib/select2/select2.js') }}"></script>
+<script src="{{ static('manage/js/supersearch_field.js') }}"></script>
+ {% endcompress %}
+
+<script>
+var ALL_PERMISSIONS = {{ all_permissions | json_dumps }};
+</script>
+{% endblock %}
+
+{% block site_css %}
+ {{ super() }}
+
+ {% compress css %}
+<link href="{{ static('crashstats/js/lib/select2/select2.css') }}" type="text/css" rel="stylesheet">
+ {% endcompress %}
+ {% compress css %}
+<link href="{{ static('manage/css/supersearch_fields.less') }}" type="text/less" rel="stylesheet">
+ {% endcompress %}
+{% endblock %}
+
+{% block mainbody %}
+
+ <div class="panel">
+ <div class="body notitle">
+
+ <form
+ id="supersearch-field"
+ method="post"
+ {% if field.name %}
+ action="{{ url('manage:supersearch_field_update') }}"
+ {% else %}
+ action="{{ url('manage:supersearch_field_create') }}"
+ {% endif %}
+ >
+ {{ csrf() }}
+ <table class="data-table vertical">
+ <tbody>
+ <tr>
+ <th scope="row">Name</th>
+ <td>
+ <input type="text" name="name" value="{{ field.name }}"{% if field.name %} readonly="readonly"{% endif %}>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Namespace</th>
+ <td>
+ <input type="text" name="namespace" value="{{ field.namespace }}">
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Name in database</th>
+ <td>
+ <input type="text" name="in_database_name" value="{{ field.in_database_name }}">
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Description</th>
+ <td>
+ <textarea name="description">{{ field.description }}</textarea>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Query type</th>
+ <td>
+ <select name="query_type" data-selected="{{ field.query_type }}">
+ <option value="enum">enum</option>
+ <option value="string">string</option>
+ <option value="number">number</option>
+ <option value="bool">bool</option>
+ <option value="date">date</option>
+ </select>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Data type</th>
+ <td>
+ <select name="data_validation_type" data-selected="{{ field.data_validation_type }}">
+ <option value="enum">enum</option>
+ <option value="str">str</option>
+ <option value="int">int</option>
+ <option value="bool">bool</option>
+ <option value="datetime">datetime</option>
+ <option value="date">date</option>
+ <option value="json">json</option>
+ </select>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Permissions needed</th>
+ <td>
+ <input type="text" name="permissions_needed" value="{{ field.permissions_needed | join(', ') }}">
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Form field type</th>
+ <td>
+ <select name="form_field_type" data-selected="{{ field.form_field_type }}">
+ <option value="MultipleValueField">MultipleValueField</option>
+ <option value="StringField">StringField</option>
+ <option value="IntegerField">IntegerField</option>
+ <option value="BooleanField">BooleanField</option>
+ <option value="DateTimeField">DateTimeField</option>
+ </select>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Form field choices</th>
+ <td>
+ <input type="text" name="form_field_choices" value="{% if field.form_field_choices %}{{ field.form_field_choices | join(', ') }}{% endif %}">
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Is exposed</th>
+ <td>
+ <input type="checkbox" name="is_exposed" {% if field.is_exposed %}checked="checkeck"{% endif %}>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Is returned</th>
+ <td>
+ <input type="checkbox" name="is_returned" {% if field.is_returned %}checked="checkeck"{% endif %}>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Has full version</th>
+ <td>
+ <input type="checkbox" name="has_full_version" {% if field.has_full_version %}checked="checkeck"{% endif %}>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Storage mapping</th>
+ <td>
+ <textarea name="storage_mapping">{% if field.storage_mapping %}{{ field.storage_mapping | json_dumps }}{% endif %}</textarea>
+ </td>
+ </tr>
+ <tr>
+ <th scope="row"></th>
+ <td>
+ {% if field.name %}
+ <button type="submit">Update</button>
+ {% else %}
+ <button type="submit">Create</button>
+ {% endif %}
+ or <a href="{{ url('manage:supersearch_fields') }}">cancel</a>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </form>
+
+ </div>
+ </div>
+
+{% endblock %}
View
93 webapp-django/crashstats/manage/templates/manage/supersearch_fields.html
@@ -0,0 +1,93 @@
+{% extends "manage/base.html" %}
+
+{% block page_title %}{{ super() }} - Super Search Fields{% endblock %}
+
+{% block admin_title %}{{ super() }} - Super Search Fields{% endblock %}
+
+{% block site_js %}
+ {{ super() }}
+
+<script type="text/javascript" src="{{ static('crashstats/js/jquery/plugins/ui/jquery.tablesorter.min.js') }}"></script>
+<script>
+$(function () {
+ $('.tablesorter').tablesorter();
+ $('.delete').click(function (e) {
+ var field_name = $(this).data('field-name');
+ return confirm('Do you really want to delete the "'+ field_name +'" field?');
+ });
+});
+</script>
+{% endblock %}
+
+{% block site_css %}
+ {{ super() }}
+
+ {% compress css %}
+<link href="{{ static('crashstats/css/flora/flora.tablesorter.css') }}" type="text/less" rel="stylesheet">
+<link href="{{ static('manage/css/supersearch_fields.less') }}" type="text/less" rel="stylesheet">
+ {% endcompress %}
+{% endblock %}
+
+{% block mainbody %}
+
+ <div class="panel">
+ <div class="body notitle">
+
+ <a href="{{ url('manage:supersearch_field') }}" class="create-field">Create a new field</a>
+
+ <p>
+ Here are all the known fields in our indexed documents.
+ This list is used by the Super Search application to determine
+ which field can be queried by the user and what data can be
+ returned. It is also used to generate the elasticsearch mapping
+ that will be used to index the crash reports we receive.
+ </p>
+
+ <table class="data-table tablesorter">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Namespace</th>
+ <th>Name in database</th>
+ <th>Description</th>
+ <th>Query type</th>
+ <th>Data type</th>
+ <th>Permissions needed</th>
+ <th>Form field type</th>
+ <th>Form field choices</th>
+ <th>Is exposed</th>
+ <th>Is returned</th>
+ <th>Has full version</th>
+ <th>Storage mapping</th>
+ <th></th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for field in fields %}
+ <tr>
+ <td>{{ field.name }}</td>
+ <td>{{ field.namespace }}</td>
+ <td>{{ field.in_database_name }}</td>
+ <td>{{ field.description }}</td>
+ <td>{{ field.query_type }}</td>
+ <td>{{ field.data_validation_type }}</td>
+ <td>{% if field.permissions_needed %}{{ field.permissions_needed | join(', ') }}{% endif %}</td>
+ <td>{{ field.form_field_type }}</td>
+ <td>{% if field.form_field_choices %}{{ field.form_field_choices | join(', ') }}{% endif %}</td>
+ <td class="boolean"><span class="{{ field.is_exposed | lower }}" title="{{ field.is_exposed }}"></span></td>
+ <td class="boolean"><span class="{{ field.is_returned | lower }}" title="{{ field.is_returned }}"></span></td>
+ <td class="boolean"><span class="{{ field.has_full_version | lower }}" title="{{ field.has_full_version }}"></span></td>
+ <td>{% if field.storage_mapping %}{{ field.storage_mapping | json_dumps }}{% endif %}</td>
+ <td>
+ <a href="{{ url('manage:supersearch_field') }}?name={{ field.name }}" class="modify" title="Modify this field">modify</a>
+ <a href="{{ url('manage:supersearch_field_delete') }}?name={{ field.name }}" class="delete" data-field-name="{{ field.name }}" title="Delete this field">delete</a>
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ </div>
+ </div>
+
+{% endblock %}
View
205 webapp-django/crashstats/manage/tests/test_views.py
<
@@ -889,3 +889,208 @@ def test_symbols_uploads(self):
eq_(response.status_code, 200)
ok_('file.zip' in response.content)
ok_('user@mozilla.com' in response.content)
+
+ @mock.patch('requests.get')
+ def test_supersearch_fields(self, rget):
+ self._login()
+ url = reverse('manage:supersearch_fields')
+
+ def mocked_get(url, **options):
+ assert '/supersearch/fields/' in url
+
+ return Response({
+ 'signature': {
+ 'name': 'signature',
+ 'namespace': 'processed_crash',
+ 'in_database_name': 'signature',
+ 'query_type': 'str',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'product': {
+ 'name': 'product',
+ 'namespace': 'processed_crash',
+ 'in_database_name': 'product',
+ 'query_type': 'enum',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ }
+ })
+
+ rget.side_effect = mocked_get
+
+ response = self.client.get(url)
+ eq_(response.status_code, 200)
+ ok_('signature' in response.content)
+ ok_('StringField' in response.content)
+ ok_('product' in response.content)
+ ok_('MultipleValueField' in response.content)
+
+ @mock.patch('requests.get')
+ def test_supersearch_field(self, rget):
+ self._login()
+ url = reverse('manage:supersearch_field')
+
+ def mocked_get(url, **options):
+ assert '/supersearch/fields/' in url
+
+ return Response({
+ 'signature': {
+ 'name': 'signature',
+ 'namespace': 'processed_crash',
+ 'in_database_name': 'signature',
+ 'query_type': 'str',
+ 'form_field_type': 'StringField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ },
+ 'platform': {
+ 'name': 'platform',
+ 'namespace': 'processed_crash',
+ 'in_database_name': 'platform',
+ 'query_type': 'enum',
+ 'form_field_type': 'MultipleValueField',
+ 'form_field_choices': None,
+ 'permissions_needed': [],
+ 'default_value': None,
+ 'is_exposed': True,
+ 'is_returned': True,
+ 'is_mandatory': False,
+ }
+ })
+
+ rget.side_effect = mocked_get
+
+ # Test when creating a new field.
+ response = self.client.get(url)
+ eq_(response.status_code, 200)
+ ok_('signature' not in response.content)
+ ok_('platform' not in response.content)
+
+ # Test when editing an existing field.
+ response = self.client.get(url, {'name': 'signature'})
+ eq_(response.status_code, 200)
+ ok_('signature' in response.content)
+ ok_('StringField' in response.content)
+ ok_('platform' not in response.content)
+
+ # Test a missing field.
+ response = self.client.get(url, {'name': 'unknown'})
+ eq_(response.status_code, 400)
+
+ @mock.patch('requests.get')
+ @mock.patch('requests.post')
+ def test_supersearch_field_create(self, rpost, rget):
+ self._login()
+ url = reverse('manage:supersearch_field_create')
+
+ def mocked_get(url, **options):
+ assert '/supersearch/fields/' in url
+ return Response({})
+
+ def mocked_post(url, data, **options):
+ assert '/supersearch/field/' in url
+ assert 'name' in data
+ assert 'in_database_name' in data
+
+ return Response(True)
+
+ rget.side_effect = mocked_get
+ rpost.side_effect = mocked_post
+
+ response = self.client.post(
+ url,
+ {
+ 'name': 'something',
+ 'in_database_name': 'something',
+ }
+ )
+ eq_(response.status_code, 302)
+
+ response = self.client.post(url)
+ eq_(response.status_code, 400)
+
+ response = self.client.post(url, {'name': 'abcd'})
+ eq_(response.status_code, 400)
+
+ response = self.client.post(url, {'in_database_name': 'bar'})
+ eq_(response.status_code, 400)
+
+ @mock.patch('requests.get')
+ @mock.patch('requests.put')
+ def test_supersearch_field_update(self, rput, rget):
+ self._login()
+ url = reverse('manage:supersearch_field_update')
+
+ def mocked_get(url, **options):
+ assert '/supersearch/fields/' in url
+ return Response({})
+
+ def mocked_put(url, data, **options):
+ assert '/supersearch/field/' in url
+ assert 'name' in data
+ assert 'description' in data
+
+ return Response(True)
+
+ rget.side_effect = mocked_get
+ rput.side_effect = mocked_put
+
+ response = self.client.post(
+ url,
+ {
+ 'name': 'something',
+ 'in_database_name': 'something',