Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixes bug 800177 - Moved old getCrash service to a new style middlewa…

…re service.
  • Loading branch information...
commit ebf5a80703d0b49c7bef38319ac4764f11d1c9b7 1 parent 97585f2
@AdrianGaudebert AdrianGaudebert authored
View
178 config/middleware.ini-dist
@@ -27,15 +27,14 @@
# name: implementation_list
# doc: list of packages for service implementations
# converter: items_list_converter
- #implementation_list='psql:socorro.external.postgresql, hbase:socorro.external.hbase, es:socorro.external.elasticsearch'
+ #implementation_list='psql:socorro.external.postgresql, hbase:socorro.external.hbase, es:socorro.external.elasticsearch, fs:socorro.external.filesystem'
# name: service_overrides
# doc: comma separated list of class overrides, e.g `Crashes: hbase`
# converter: items_list_converter
# An example of this would be something like
# 'Crashes: hbase, SignatureSummary: es'
- #service_overrides=''
-
+ #service_overrides='CrashData:fs'
[database]
@@ -97,6 +96,135 @@
# converter: configman.converters.class_converter
#transaction_executor_class='socorro.database.transaction_executor.TransactionExecutorWithLimitedBackoff'
+[filesystem]
+
+ # name: date_name
+ # doc: the relative path to the top of the date storage tree from root parameter
+ # converter: str
+ #date_name='date'
+
+ # name: def_fs_root
+ # doc: a path to a local file system
+ # converter: str
+ #def_fs_root='./deferredCrashStore'
+
+ # name: dir_permissions
+ # doc: a number used for permissions for directories in the local file system
+ # converter: int
+ #dir_permissions='504'
+
+ # name: dump_dir_count
+ # doc: the number of dumps to be stored in a single directory in the local file system
+ # converter: int
+ #dump_dir_count='1024'
+
+ # name: dump_file_suffix
+ # doc: the suffix used to identify a dump file
+ # converter: str
+ #dump_file_suffix='.dump'
+
+ # name: dump_gid
+ # doc: the group ID for saved crashes in local file system (optional)
+ # converter: str
+ #dump_gid=''
+
+ # name: dump_permissions
+ # doc: a number used for permissions crash dump files in the local file system
+ # converter: int
+ #dump_permissions='432'
+
+ # name: filesystem_class
+ # doc: None
+ # converter: configman.converters.class_converter
+ #filesystem_class='socorro.external.filesystem.crashstorage.FileSystemCrashStorage'
+
+ # name: forbidden_keys
+ # doc: a comma delimited list of keys to not allowed in the processed crash
+ # converter: socorro.external.filesystem.crashstorage.<lambda>
+ #forbidden_keys='url, email, user_id'
+
+ # name: gzip_compression_level
+ # doc: the level of compression to use
+ # converter: int
+ #gzip_compression_level='9'
+
+ # name: index_name
+ # doc: the relative path to the top of the name storage tree from root parameter
+ # converter: str
+ #index_name='name'
+
+ # name: json_file_suffix
+ # doc: the suffix used to identify a json file
+ # converter: str
+ #json_file_suffix='.json'
+
+ # name: minutes_per_slot
+ # doc: the number of minutes in the lowest date directory
+ # converter: int
+ #minutes_per_slot='1'
+
+ # name: pro_fs_root
+ # doc: a path to a local file system for processed storage
+ # converter: str
+ #pro_fs_root='./processedCrashStore'
+
+ # name: processed_crash_file_suffix
+ # doc: the processed crash filename suffix
+ # converter: str
+ #processed_crash_file_suffix='.jsonz'
+
+ # name: std_fs_root
+ # doc: a path to a local file system
+ # converter: str
+ #std_fs_root='./primaryCrashStore'
+
+ # name: storage_depth
+ # doc: the length of branches in the radix storage tree
+ # converter: int
+ #storage_depth='2'
+
+ # name: sub_slot_count
+ # doc: distribute data evenly among this many sub timeslots
+ # converter: int
+ #sub_slot_count='1'
+
+[hbase]
+
+ # name: forbidden_keys
+ # doc: a comma delimited list of keys banned from the processed crash in HBase
+ # converter: socorro.external.hbase.crashstorage.<lambda>
+ #forbidden_keys='email, url, user_id'
+
+ # name: hbase_class
+ # doc: None
+ # converter: configman.converters.class_converter
+ #hbase_class='socorro.external.hbase.crashstorage.HBaseCrashStorage'
+
+ # name: hbase_host
+ # doc: Host to HBase server
+ # converter: str
+ #hbase_host='localhost'
+
+ # name: hbase_port
+ # doc: Port to HBase server
+ # converter: int
+ #hbase_port='9090'
+
+ # name: hbase_timeout
+ # doc: timeout in milliseconds for an HBase connection
+ # converter: int
+ #hbase_timeout='5000'
+
+ # name: number_of_retries
+ # doc: Max. number of retries when fetching from hbaseClient
+ # converter: int
+ #number_of_retries='0'
+
+ # name: transaction_executor_class
+ # doc: a class that will execute transactions
+ # converter: configman.converters.class_converter
+ #transaction_executor_class='socorro.database.transaction_executor.TransactionExecutor'
+
[logging]
# if you'd like to have one common logging configuration for the
@@ -142,3 +270,47 @@
# converter: int
# OPS - set once
#syslog_port='514'
+
+[revisions]
+
+ # name: breakpad_revision
+ # doc: the current revision of Breakpad
+ # converter: str
+ breakpad_revision='CURRENT_BREAKPAD_REVISION'
+
+ # name: socorro_revision
+ # doc: the current revision of Socorro
+ # converter: str
+ socorro_revision='CURRENT_SOCORRO_REVISION'
+
+[webapi]
+
+ # name: channels
+ # doc: List of release channels, excluding the `release` one.
+ # converter: string_to_list
+ #channels='Beta, Aurora, Nightly, beta, aurora, nightly'
+
+ # name: elasticSearchHostname
+ # doc: String containing the URI of the Elastic Search instance.
+ # converter: str
+ #elasticSearchHostname='localhost'
+
+ # name: elasticSearchPort
+ # doc: String containing the port on which calling the Elastic Search instance.
+ # converter: str
+ #elasticSearchPort='9200'
+
+ # name: platforms
+ # doc: Array associating OS ids to full names.
+ # converter: <lambda>
+ #platforms='{"id": "windows", "name": "Windows NT"}, {"id": "mac", "name": "Mac OS X"}, {"id": "linux", "name": "Linux"}'
+
+ # name: restricted_channels
+ # doc: List of release channels to restrict based on build ids.
+ # converter: string_to_list
+ #restricted_channels='Beta, beta'
+
+ # name: searchMaxNumberOfDistinctSignatures
+ # doc: Integer containing the maximum allowed number of distinct signatures the system should retrieve. Used mainly for performances in ElasticSearch
+ # converter: int
+ #searchMaxNumberOfDistinctSignatures='1000'
View
153 docs/middleware.rst
@@ -12,43 +12,40 @@ New-style, documented services
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
* `/bugs/ <#bugs>`_
+* `/crash/ <#crash>`_
+* `/crash_data/ <#crash-data>`_
* /crashes/
* `/crashes/comments <#crashes-comments>`_
* `/crashes/daily <#crashes-daily>`_
* `/crashes/frequency <#crashes-frequency>`_
* `/crashes/paireduuid <#crashes-paireduuid>`_
* `/crashes/signatures <#crashes-signatures>`_
-* `extensions/ <#id7>`_
-* `crashtrends/ <#crashtrends>`_
-* `job/ <#job>`_
-* `platforms/ <#platforms>`_
-* `priorityjobs/ <#priorityjobs>`_
-* `products/ <#products>`_
-* `products/builds/ <#products-builds>`_
-* products/
- * `products/builds/ <#products-builds>`_
- * `products/versions/ <#products-versions>`_
-* releases/
- * `releases/featured/ <#releases-featured>`_
-* report/
- * `report/list/ <#list-report>`_
-* `signatureurls <#signature-urls>`_
-* search/
- * `search/crashes/ <#search>`_
- * `search/signatures/ <#search>`_
-* `server_status/ <#server-status>`_
-* util/
- * `util/versions_info/ <#versions-info>`_
+* `/crashtrends/ <#crashtrends>`_
+* `/extensions/ <#extensions>`_
+* `/job/ <#job>`_
+* `/platforms/ <#platforms>`_
+* `/priorityjobs/ <#priorityjobs>`_
+* `/products/ <#products>`_
+* /products/
+ * `/products/builds/ <#products-builds>`_
+ * `/products/versions/ <#products-versions>`_
+* /releases/
+ * `/releases/featured/ <#releases-featured>`_
+* /report/
+ * `/report/list/ <#list-report>`_
+* `/signatureurls <#signature-urls>`_
+* /search/
+ * `/search/crashes/ <#search>`_
+ * `/search/signatures/ <#search>`_
+* `/server_status/ <#server-status>`_
+* /util/
+ * `/util/versions_info/ <#versions-info>`_
Old-style, undocumented services
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
See source code in ``.../socorro/services/`` for more details.
-* /adu/byday
-* /adu/byday/details
-* /bugs/by/signatures
-* /crash
* /current/versions
* /email
* /emailcampaigns/campaign
@@ -61,6 +58,7 @@ See source code in ``.../socorro/services/`` for more details.
* /topcrash/sig/trend/history
* /topcrash/sig/trend/rank
+
.. ############################################################################
Bugs API
############################################################################
@@ -119,6 +117,111 @@ In normal cases, return something like this::
.. ############################################################################
+ Crash API
+ ############################################################################
+
+Crash
+-----
+
+Return a single crash report from its UUID.
+
+API specifications
+^^^^^^^^^^^^^^^^^^
+
++----------------+-----------------------------------------------------------------------------------+
+| HTTP method | POST |
++----------------+-----------------------------------------------------------------------------------+
+| URL schema | /crash/(optional_parameters) |
++----------------+-----------------------------------------------------------------------------------+
+| Full URL | /crash/uuid/(uuid)/ |
++----------------+-----------------------------------------------------------------------------------+
+| Example | http://socorro-api/bpapi/crash/uuid/58727744-12f5-454a-bcf5-f688af393821/ |
++----------------+-----------------------------------------------------------------------------------+
+
+Mandatory parameters
+^^^^^^^^^^^^^^^^^^^^
+
++----------------+------------------+---------------+-------------------------+
+| Name | Type of value | Default value | Description |
++================+==================+===============+=========================+
+| uuid | String | None | Identifier of the crash |
+| | | | report to get. |
++----------------+------------------+---------------+-------------------------+
+
+Optional parameters
+^^^^^^^^^^^^^^^^^^^
+
+None.
+
+Return value
+^^^^^^^^^^^^
+
+In normal cases, return something like this::
+
+ {
+ "hits": [
+ {
+ "email": "someone@example.com",
+ "url": "http://example.com/somepage",
+ "addons_checked": "some addons",
+ "exploitability": "high",
+ "duplicate_of": 123456
+ }
+ ],
+ "total": 1
+ }
+
+
+.. ############################################################################
+ Crash Data API
+ ############################################################################
+
+Crash Data
+----------
+
+Return JSON or binary data of a crash report, given its uuid.
+
+API specifications
+^^^^^^^^^^^^^^^^^^
+
++----------------+---------------------------------------------------------------------------------------------+
+| HTTP method | POST |
++----------------+---------------------------------------------------------------------------------------------+
+| URL schema | /crash_data/(optional_parameters) |
++----------------+---------------------------------------------------------------------------------------------+
+| Full URL | /crash_data/datatype/(datatype)/uuid/(uuid)/ |
++----------------+---------------------------------------------------------------------------------------------+
+| Example | http://socorro-api/bpapi/crash_data/datatype/raw/uuid/58727744-12f5-454a-bcf5-f688af393821/ |
++----------------+---------------------------------------------------------------------------------------------+
+
+Mandatory parameters
+^^^^^^^^^^^^^^^^^^^^
+
++----------------+------------------+---------------+-------------------------+
+| Name | Type of value | Default value | Description |
++================+==================+===============+=========================+
+| datatype | String | None | Type of data to get, can|
+| | | | be 'raw', 'meta' or |
+| | | | 'processed'. |
++----------------+------------------+---------------+-------------------------+
+| uuid | String | None | Identifier of the crash |
+| | | | report to get. |
++----------------+------------------+---------------+-------------------------+
+
+Optional parameters
+^^^^^^^^^^^^^^^^^^^
+
+None.
+
+Return value
+^^^^^^^^^^^^
+
+If datatype is 'raw', returns the binary raw dump of the crash report.
+If datatype is 'meta', returns the raw JSON of the crash report.
+If datatype is 'processed', return the processed JSON of the crash report.
+
+
+.. ############################################################################
Crashes Comments API
############################################################################
View
6 puppet/files/etc_apache2_sites-available/crash-stats
@@ -6,13 +6,13 @@
ReWriteEngine On
RewriteCond %{REQUEST_URI} /dumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).jsonz
- ReWriteRule /dumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).jsonz http://socorro-api/crashes/crash/processed/by/uuid/$1$2$3$4 [P]
+ ReWriteRule /dumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).jsonz http://socorro-api/crashes/crash_data/datatype/processed/uuid/$1$2$3$4 [P]
RewriteCond %{REQUEST_URI} /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).dump
- ReWriteRule /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).dump http://socorro-api/crashes/crash/raw_crash/by/uuid/$1$2$3$4 [P]
+ ReWriteRule /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).dump http://socorro-api/crashes/crash_data/datatype/raw_crash/uuid/$1$2$3$4 [P]
RewriteCond %{REQUEST_URI} /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).json
- ReWriteRule /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).json http://socorro-api/crashes/crash/meta/by/uuid/$1$2$3$4 [P]
+ ReWriteRule /rawdumps/(\w\w)(\w\w)((?:\w|-)+?)(\d{6}).json http://socorro-api/crashes/crash_data/datatype/meta/uuid/$1$2$3$4 [P]
ReWriteRule /rawdumps/(.*).(json|dump) /missing_dump [L]
View
10 scripts/config/webapiconfig.py.dist
@@ -100,6 +100,12 @@ crashImplementationModule.doc = (
"String, name of the module crash service uses.")
crashImplementationModule.default = 'socorro.external.postgresql'
+# CrashData service config
+crash_dataImplementationModule = cm.Option()
+crash_dataImplementationModule.doc = (
+ "String, name of the module crash_data service uses.")
+crash_dataImplementationModule.default = 'socorro.external.hbase'
+
# Extensions service config
extensionsImplementationModule = cm.Option()
extensionsImplementationModule.doc = (
@@ -114,7 +120,6 @@ priorityjobsImplementationModule.default = 'socorro.external.postgresql'
import socorro.services.signatureHistory as sighist
-import socorro.services.getCrash as crash
import socorro.services.emailCampaign as emailcampaign
import socorro.services.emailCampaignCreate as emailcreate
import socorro.services.emailCampaigns as emaillist
@@ -133,6 +138,7 @@ import socorro.middleware.crash_trends_service as crash_trends
import socorro.middleware.tcbs_service as tcbs
import socorro.middleware.crashes_comments_service as crashes_comments
import socorro.middleware.crash_service as crash_new
+import socorro.middleware.crash_data_service as crash_data
import socorro.middleware.extensions_service as extensions
import socorro.middleware.crashes_paireduuid_service as crashes_paireduuid
import socorro.middleware.products_service as products
@@ -149,7 +155,6 @@ servicesList = cm.Option()
servicesList.doc = 'a python list of classes to offer as services'
servicesList.default = [
sighist.SignatureHistory,
- crash.GetCrash,
emailcampaign.EmailCampaign,
emailcreate.EmailCampaignCreate,
emaillist.EmailCampaigns,
@@ -168,6 +173,7 @@ servicesList.default = [
tcbs.TCBS,
crashes_comments.CrashesComments,
crash_new.Crash,
+ crash_data.CrashData,
extensions.Extensions,
crashes_paireduuid.CrashesPaireduuid,
products.Products,
View
11 socorro/external/__init__.py
@@ -19,3 +19,14 @@ class DatabaseError(Exception):
class MissingOrBadArgumentError(Exception):
"""When a mandatory argument is missing or has a bad value. """
pass
+
+
+class ResourceNotFound(Exception):
+ """When a resource could not be found in a storage system. """
+ pass
+
+
+class ResourceUnavailable(Exception):
+ """When a resource could not be found in a storage system because it is
+ not ready yet (but could be accessible later). """
+ pass
View
60 socorro/external/filesystem/crash_data.py
@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from socorro.external import MissingOrBadArgumentError, ResourceNotFound, \
+ ResourceUnavailable
+from socorro.external.crashstorage_base import CrashIDNotFound
+from socorro.external.postgresql import priorityjobs
+from socorro.lib import external_common
+
+import crashstorage
+
+
+class CrashData(object):
+
+ """
+ Implement the /crash_data service with the file system.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(CrashData, self).__init__()
+ self.config = kwargs['config']
+
+ def get(self, **kwargs):
+ """Return JSON data of a crash report, given its uuid. """
+ filters = [
+ ('uuid', None, 'str'),
+ ('datatype', None, 'str')
+ ]
+ params = external_common.parse_arguments(filters, kwargs)
+
+ if not params.uuid:
+ raise MissingOrBadArgumentError(
+ "Mandatory parameter 'uuid' is missing or empty")
+
+ if not params.datatype:
+ raise MissingOrBadArgumentError(
+ "Mandatory parameter 'datatype' is missing or empty")
+
+ store = crashstorage.FileSystemCrashStorage(self.config.filesystem)
+
+ datatype_method_mapping = {
+ 'raw': 'get_raw_dump',
+ 'meta': 'get_raw_crash',
+ 'processed': 'get_processed'
+ }
+
+ get = store.__getattribute__(datatype_method_mapping[params.datatype])
+ try:
+ if params.datatype == 'raw':
+ return (get(params.uuid), 'application/octet-stream')
+ else:
+ return get(params.uuid)
+ except CrashIDNotFound:
+ if params.datatype == 'processed':
+ self.get(datatype='raw', uuid=params.uuid)
+ j = priorityjobs.Priorityjobs(config=self.config)
+ j.create(uuid=params.uuid)
+ raise ResourceUnavailable(params.uuid)
+ raise ResourceNotFound(params.uuid)
View
6 socorro/external/filesystem/crashstorage.py
@@ -41,7 +41,7 @@ class FileSystemRawCrashStorage(CrashStorageBase):
required_config.add_option(
'std_fs_root',
doc='a path to a local file system',
- default='/home/socorro/primaryCrashStore'
+ default='./primaryCrashStore'
)
required_config.add_option(
'dump_dir_count',
@@ -208,7 +208,7 @@ class FileSystemThrottledCrashStorage(FileSystemRawCrashStorage):
required_config.add_option(
'def_fs_root',
doc='a path to a local file system',
- default='/home/socorro/deferredCrashStore'
+ default='./deferredCrashStore'
)
#--------------------------------------------------------------------------
@@ -329,7 +329,7 @@ class FileSystemCrashStorage(FileSystemThrottledCrashStorage):
required_config.add_option(
'pro_fs_root',
doc='a path to a local file system for processed storage',
- default='/home/socorro/processedCrashStore'
+ default='./processedCrashStore'
)
required_config.add_option(
'minutes_per_slot',
View
82 socorro/external/hbase/crash_data.py
@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+from socorro.external import MissingOrBadArgumentError, ResourceNotFound, \
+ ResourceUnavailable
+from socorro.external.crashstorage_base import CrashIDNotFound
+from socorro.external.postgresql import priorityjobs
+from socorro.lib import external_common
+
+from . import crashstorage
+from hbase_client import OoidNotFoundException
+
+logger = logging.getLogger("webapi")
+
+
+class CrashData(object):
+
+ """
+ Implement the /crash_data service with HBase.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(CrashData, self).__init__()
+ self.config = kwargs["config"]
+
+ def get(self, **kwargs):
+ """Return JSON data of a crash report, given its uuid. """
+ filters = [
+ ("uuid", None, "str"),
+ ("datatype", None, "str")
+ ]
+ params = external_common.parse_arguments(filters, kwargs)
+
+ if not params.uuid:
+ raise MissingOrBadArgumentError(
+ "Mandatory parameter 'uuid' is missing or empty")
+
+ if not params.datatype:
+ raise MissingOrBadArgumentError(
+ "Mandatory parameter 'datatype' is missing or empty")
+
+ if hasattr(self.config, 'hbase'):
+ config = self.config.hbase
+ store = crashstorage.HBaseCrashStorage(config)
+
+ datatype_method_mapping = {
+ "raw": "get_raw_dump",
+ "meta": "get_raw_crash",
+ "processed": "get_processed"
+ }
+
+ else:
+ # old middleware
+ config = self.config
+ import socorro.storage.crashstorage as cs
+ store = cs.CrashStoragePool(
+ config,
+ storageClass=config.hbaseStorageClass
+ ).crashStorage()
+
+ datatype_method_mapping = {
+ "raw": "get_raw_dump",
+ "meta": "get_meta",
+ "processed": "get_processed"
+ }
+
+ get = store.__getattribute__(datatype_method_mapping[params.datatype])
+ try:
+ if params.datatype == 'raw':
+ return (get(params.uuid), 'application/octet-stream')
+ else:
+ return get(params.uuid)
+ except (CrashIDNotFound, OoidNotFoundException):
+ if params.datatype == 'processed':
+ self.get(datatype='raw', uuid=params.uuid)
+ j = priorityjobs.Priorityjobs(config=self.config)
+ j.create(uuid=params.uuid)
+ raise ResourceUnavailable(params.uuid)
+ raise ResourceNotFound(params.uuid)
View
40 socorro/middleware/crash_data_service.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import web
+
+from socorro.external import ResourceNotFound, ResourceUnavailable
+from socorro.middleware.service import DataAPIService
+from socorro.webapi import webapiService
+
+logger = logging.getLogger("webapi")
+
+
+class CrashData(DataAPIService):
+
+ """
+ Return JSON data of a crash report, given its uuid.
+ """
+
+ service_name = "crash_data"
+ uri = "/crash_data/(.*)"
+
+ def __init__(self, config):
+ super(CrashData, self).__init__(config)
+ logger.debug('CrashData service __init__')
+
+ def get(self, *args):
+ """
+ Called when a get HTTP request is executed to /crash_data
+ """
+ params = self.parse_query_string(args[0])
+ module = self.get_module(params)
+ impl = module.CrashData(config=self.context)
+ try:
+ return impl.get(**params)
+ except ResourceNotFound:
+ raise web.webapi.NotFound()
+ except ResourceUnavailable:
+ raise webapiService.Timeout()
View
40 socorro/middleware/middleware_app.py
@@ -14,8 +14,11 @@
import json
import web
from socorro.app.generic_app import App, main
-from socorro.external import MissingOrBadArgumentError
-from socorro.webapi.webapiService import JsonWebServiceBase
+from socorro.external import MissingOrBadArgumentError, ResourceNotFound, \
+ ResourceUnavailable
+from socorro.webapi.webapiService import JsonWebServiceBase, Timeout
+from socorro.external.filesystem.crashstorage import FileSystemCrashStorage
+from socorro.external.hbase.crashstorage import HBaseCrashStorage
from socorro.external.postgresql.connection_context import ConnectionContext
from configman import Namespace
@@ -26,6 +29,7 @@
# The final lookup depends on the `implementation_list` option inside the app.
SERVICES_LIST = (
(r'/bugs/', 'bugs.Bugs'),
+ (r'/crash_data/(.*)', 'crash_data.CrashData'),
(r'/crash/(.*)', 'crash.Crash'),
(r'/crashes/(comments|daily|frequency|paireduuid|signatures)/(.*)',
'crashes.Crashes'),
@@ -110,14 +114,15 @@ class MiddlewareApp(App):
doc='list of packages for service implementations',
default='psql:socorro.external.postgresql, '
'hbase:socorro.external.hbase, '
- 'es:socorro.external.elasticsearch',
+ 'es:socorro.external.elasticsearch, '
+ 'fs:socorro.external.filesystem',
from_string_converter=items_list_converter
)
required_config.implementations.add_option(
'service_overrides',
doc='comma separated list of class overrides, e.g `Crashes: hbase`',
- default='', # e.g. 'Crashes: es',
+ default='CrashData: fs', # e.g. 'Crashes: es',
from_string_converter=items_list_converter
)
@@ -133,6 +138,28 @@ class MiddlewareApp(App):
)
#--------------------------------------------------------------------------
+ # hbase namespace
+ # the namespace is for external implementations of the services
+ #-------------------------------------------------------------------------
+ required_config.namespace('hbase')
+ required_config.hbase.add_option(
+ 'hbase_class',
+ default=HBaseCrashStorage,
+ from_string_converter=class_converter
+ )
+
+ #--------------------------------------------------------------------------
+ # filesystem namespace
+ # the namespace is for external implementations of the services
+ #-------------------------------------------------------------------------
+ required_config.namespace('filesystem')
+ required_config.filesystem.add_option(
+ 'filesystem_class',
+ default=FileSystemCrashStorage,
+ from_string_converter=class_converter
+ )
+
+ #--------------------------------------------------------------------------
# webapi namespace
# this is all config options that used to belong to webapiconfig.py
#-------------------------------------------------------------------------
@@ -220,7 +247,6 @@ class MiddlewareApp(App):
from socorro.webapi.servers import StandAloneServer
StandAloneServer.required_config.port.set_default(8883, force=True)
-
#--------------------------------------------------------------------------
def main(self):
# Apache modwsgi requireds a module level name 'applicaiton'
@@ -326,6 +352,10 @@ def GET(self, *args, **kwargs):
except MissingOrBadArgumentError, msg:
raise BadRequest(str(msg))
+ except ResourceNotFound, msg:
+ raise web.webapi.NotFound(str(msg))
+ except ResourceUnavailable, msg:
+ raise Timeout(str(msg))
def POST(self, *args, **kwargs):
params = self._get_web_input_params()
View
62 socorro/services/getCrash.py
@@ -1,62 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import logging
-logger = logging.getLogger("webapi")
-
-import socorro.lib.util as util
-import socorro.webapi.webapiService as webapi
-import web
-
-from socorro.external.postgresql.priorityjobs import Priorityjobs
-
-datatype_options = ('meta', 'raw_crash', 'processed')
-crashStorageFunctions = ('fetchMeta', 'fetchRaw', 'fetchProcessed')
-datatype_function_associations = dict(zip(datatype_options, crashStorageFunctions))
-
-class NotADataTypeOption(Exception):
- def __init__(self, reason):
- #super(NotADataTypeOption, self).__init__("%s must be one of %s" % (reason, ','.join(datatype_options))
- Exception.__init__("%s must be one of %s" % (reason, ','.join(datatype_options)))
-
-def dataTypeOptions(x):
- if x in datatype_options:
- return x
- raise NotADataTypeOption(x)
-
-#=================================================================================================================
-class GetCrash(webapi.JsonServiceBase):
- #-----------------------------------------------------------------------------------------------------------------
- def __init__(self, configContext):
- super(GetCrash, self).__init__(configContext)
- logger.debug('GetCrash __init__')
- #-----------------------------------------------------------------------------------------------------------------
- uri = '/crash/(.*)/by/uuid/(.*)'
- #-----------------------------------------------------------------------------------------------------------------
- def get(self, *args):
- convertedArgs = webapi.typeConversion([dataTypeOptions,str], args)
- parameters = util.DotDict(zip(['datatype','uuid'], convertedArgs))
- logger.debug("GetCrash get %s", parameters)
- self.crashStorage = self.crashStoragePool.crashStorage()
- function_name = datatype_function_associations[parameters.datatype]
- function = self.__getattribute__(function_name)
- return function(parameters.uuid)
-
- def fetchProcessed(self, uuid):
- try:
- return self.crashStorage.get_processed(uuid)
- except Exception:
- try:
- raw = self.fetchRaw(uuid)
- j = Priorityjobs(config=self.context)
- j.create(uuid=uuid)
- except Exception:
- raise web.webapi.NotFound()
- raise webapi.Timeout()
-
- def fetchMeta(self, uuid):
- return self.crashStorage.get_meta(uuid)
-
- def fetchRaw(self, uuid):
- return (self.crashStorage.get_raw_dump(uuid), "application/octet-stream")
View
154 socorro/unittest/external/filesystem/test_crash_data.py
@@ -0,0 +1,154 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import shutil
+import tempfile
+import unittest
+from configman import ConfigurationManager, Namespace
+from mock import Mock, patch
+from nose.plugins.attrib import attr
+
+from socorro.external import MissingOrBadArgumentError, ResourceNotFound, \
+ ResourceUnavailable
+from socorro.external.filesystem import crash_data, crashstorage
+
+
+@attr(integration='filesystem') # for nosetests
+class IntegrationTestCrashData(unittest.TestCase):
+
+ def setUp(self):
+ """Insert fake data into filesystem. """
+ self.std_tmp_dir = tempfile.mkdtemp()
+ self.def_tmp_dir = tempfile.mkdtemp()
+ self.pro_tmp_dir = tempfile.mkdtemp()
+
+ self.config_manager = self._common_config_setup()
+
+ with self.config_manager.context() as config:
+ store = crashstorage.FileSystemCrashStorage(config.filesystem)
+ fake_dump = 'this is a fake dump'
+ fake_raw = {'name': 'Peter', 'legacy_processing': 0}
+ fake_processed = {
+ 'name': 'Peter',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'
+ }
+
+ store.save_raw_crash(
+ fake_raw,
+ fake_dump,
+ '114559a5-d8e6-428c-8b88-1c1f22120314'
+ )
+ store.save_processed(fake_processed)
+
+ fake_dump = 'this is another fake dump'
+ fake_raw = {'name': 'Adrian', 'legacy_processing': 0}
+
+ store.save_raw_crash(
+ fake_raw,
+ fake_dump,
+ '58727744-12f5-454a-bcf5-f688af393821'
+ )
+
+ def tearDown(self):
+ """Remove all temp files and folders. """
+ shutil.rmtree(self.std_tmp_dir)
+ shutil.rmtree(self.def_tmp_dir)
+ shutil.rmtree(self.pro_tmp_dir)
+
+ def _common_config_setup(self):
+ mock_logging = Mock()
+ required_config = Namespace()
+ required_config.namespace('filesystem')
+ required_config.filesystem = \
+ crashstorage.FileSystemCrashStorage.get_required_config()
+ required_config.filesystem.add_option('logger', default=mock_logging)
+ config_manager = ConfigurationManager(
+ [required_config],
+ app_name='testapp',
+ app_version='1.0',
+ app_description='app description',
+ values_source_list=[{'filesystem': {
+ 'logger': mock_logging,
+ 'std_fs_root': self.std_tmp_dir,
+ 'def_fs_root': self.def_tmp_dir,
+ 'pro_fs_root': self.pro_tmp_dir,
+ }}]
+ )
+ return config_manager
+
+ @patch('socorro.external.postgresql.priorityjobs.Priorityjobs')
+ def test_get(self, priorityjobs_mock):
+ with self.config_manager.context() as config:
+ service = crash_data.CrashData(config=config)
+ params = {
+ 'datatype': 'raw',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'
+ }
+
+ # Test 1: get a raw dump
+ res_expected = ('this is a fake dump', 'application/octet-stream')
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 2: get a raw crash
+ params['datatype'] = 'meta'
+ res_expected = {'name': 'Peter', 'legacy_processing': 0}
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 3: get a processed crash
+ params['datatype'] = 'processed'
+ res_expected = {
+ 'name': 'Peter',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'
+ }
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 4: missing parameters
+ self.assertRaises(
+ MissingOrBadArgumentError,
+ service.get
+ )
+ self.assertRaises(
+ MissingOrBadArgumentError,
+ service.get,
+ **{'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'}
+ )
+
+ # Test 5: crash cannot be found
+ self.assertRaises(
+ ResourceNotFound,
+ service.get,
+ **{
+ 'uuid': 'c44245f4-c93b-49b8-86a2-c15dc3a695cb',
+ 'datatype': 'processed'
+ }
+ )
+
+ # Test 6: not yet available crash
+ self.assertRaises(
+ ResourceUnavailable,
+ service.get,
+ **{
+ 'uuid': '58727744-12f5-454a-bcf5-f688af393821',
+ 'datatype': 'processed'
+ }
+ )
+ priorityjobs_mock.return_value.create.assert_called_once_with(
+ uuid='58727744-12f5-454a-bcf5-f688af393821'
+ )
+
+ # Test 7: raw crash cannot be found
+ self.assertRaises(
+ ResourceNotFound,
+ service.get,
+ **{
+ 'uuid': 'c44245f4-c93b-49b8-86a2-c15dc3a695cb',
+ 'datatype': 'raw'
+ }
+ )
View
187 socorro/unittest/external/hbase/test_crash_data.py
@@ -0,0 +1,187 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import unittest
+from configman import ConfigurationManager, Namespace
+from mock import Mock, patch
+from nose.plugins.attrib import attr
+
+from socorro.external import MissingOrBadArgumentError, ResourceNotFound, \
+ ResourceUnavailable
+from socorro.external.hbase import crash_data, crashstorage, hbase_client
+
+
+_run_integration_tests = os.environ.get('RUN_HBASE_INTEGRATION_TESTS', False)
+if _run_integration_tests in ('false', 'False', 'no', '0'):
+ _run_integration_tests = False
+
+
+if not _run_integration_tests:
+ import logging
+ logging.warning("Skipping HBase integration tests")
+
+else:
+
+ @attr(integration='hbase') # for nosetests
+ class TestIntegrationHBaseCrashData(unittest.TestCase):
+
+ def setUp(self):
+ self.config_manager = self._common_config_setup()
+
+ with self.config_manager.context() as config:
+ store = crashstorage.HBaseCrashStorage(config.hbase)
+
+ # A complete crash report (raw, dump and processed)
+ fake_raw_dump_1 = 'peter is a swede'
+ fake_raw_dump_2 = 'lars is a norseman'
+ fake_raw_dump_3 = 'adrian is a frenchman'
+ fake_dumps = {'upload_file_minidump': fake_raw_dump_1,
+ 'lars': fake_raw_dump_2,
+ 'adrian': fake_raw_dump_3}
+ fake_raw = {
+ 'name': 'Peter',
+ 'legacy_processing': 0,
+ 'submitted_timestamp': '1234567890'
+ }
+ fake_processed = {
+ 'name': 'Peter',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314',
+ 'completeddatetime': '2012-01-01T00:00:00'
+ }
+
+ store.save_raw_crash(
+ fake_raw,
+ fake_dumps,
+ '114559a5-d8e6-428c-8b88-1c1f22120314'
+ )
+ store.save_processed(fake_processed)
+
+ # A non-processed crash report
+ fake_raw = {
+ 'name': 'Adrian',
+ 'legacy_processing': 0,
+ 'submitted_timestamp': '1234567890'
+ }
+
+ store.save_raw_crash(
+ fake_raw,
+ fake_dumps,
+ '58727744-12f5-454a-bcf5-f688af393821'
+ )
+
+ def tearDown(self):
+ with self.config_manager.context() as config:
+ connection = hbase_client.HBaseConnectionForCrashReports(
+ config.hbase.hbase_host,
+ config.hbase.hbase_port,
+ config.hbase.hbase_timeout
+ )
+ for row in connection.merge_scan_with_prefix(
+ 'crash_reports', '', ['ids:ooid']):
+ index_row_key = row['_rowkey']
+ connection.client.deleteAllRow(
+ 'crash_reports', index_row_key)
+ # because of HBase's async nature, deleting can take time
+ list(connection.iterator_for_all_legacy_to_be_processed())
+
+ def _common_config_setup(self):
+ mock_logging = Mock()
+ required_config = Namespace()
+ required_config.namespace('hbase')
+ required_config.hbase = \
+ crashstorage.HBaseCrashStorage.get_required_config()
+ required_config.hbase.add_option('logger', default=mock_logging)
+ config_manager = ConfigurationManager(
+ [required_config],
+ app_name='testapp',
+ app_version='1.0',
+ app_description='app description',
+ values_source_list=[{'hbase': {
+ 'logger': mock_logging
+ }}]
+ )
+ return config_manager
+
+ @patch('socorro.external.postgresql.priorityjobs.Priorityjobs')
+ def test_get(self, priorityjobs_mock):
+ with self.config_manager.context() as config:
+ service = crash_data.CrashData(config=config)
+ params = {
+ 'datatype': 'raw',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'
+ }
+
+ # Test 1: get a raw dump
+ res_expected = ('peter is a swede',
+ 'application/octet-stream')
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 2: get a raw crash
+ params['datatype'] = 'meta'
+ res_expected = {
+ 'name': 'Peter',
+ 'legacy_processing': 0,
+ 'submitted_timestamp': '1234567890'
+ }
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 3: get a processed crash
+ params['datatype'] = 'processed'
+ res_expected = {
+ 'name': 'Peter',
+ 'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314',
+ 'completeddatetime': '2012-01-01T00:00:00'
+ }
+ res = service.get(**params)
+
+ self.assertEqual(res, res_expected)
+
+ # Test 4: missing parameters
+ self.assertRaises(
+ MissingOrBadArgumentError,
+ service.get
+ )
+ self.assertRaises(
+ MissingOrBadArgumentError,
+ service.get,
+ **{'uuid': '114559a5-d8e6-428c-8b88-1c1f22120314'}
+ )
+
+ # Test 5: crash cannot be found
+ self.assertRaises(
+ ResourceNotFound,
+ service.get,
+ **{
+ 'uuid': 'c44245f4-c93b-49b8-86a2-c15dc3a695cb',
+ 'datatype': 'processed'
+ }
+ )
+
+ # Test 6: not yet available crash
+ self.assertRaises(
+ ResourceUnavailable,
+ service.get,
+ **{
+ 'uuid': '58727744-12f5-454a-bcf5-f688af393821',
+ 'datatype': 'processed'
+ }
+ )
+ priorityjobs_mock.return_value.create.assert_called_once_with(
+ uuid='58727744-12f5-454a-bcf5-f688af393821'
+ )
+
+ # Test 7: raw crash cannot be found
+ self.assertRaises(
+ ResourceNotFound,
+ service.get,
+ **{
+ 'uuid': 'c44245f4-c93b-49b8-86a2-c15dc3a695cb',
+ 'datatype': 'raw'
+ }
+ )
View
6 socorro/unittest/middleware/test_middleware_app.py
@@ -329,7 +329,7 @@ def put(self, server, url, data=None, expect_errors=False):
def test_overriding_implementation_class(self):
config_manager = self._setup_config_manager({
- 'implementations.service_overrides': 'Crash: typo'
+ 'implementations.service_overrides': 'CrashData: fs, Crash: typo'
})
with config_manager.context() as config:
@@ -344,7 +344,7 @@ def test_overriding_implementation_class(self):
previous_as_str = ', '.join('%s: %s' % (x, y) for (x, y) in default)
config_manager = self._setup_config_manager({
- 'implementations.service_overrides': 'Crash: testy',
+ 'implementations.service_overrides': 'CrashData: fs, Crash: testy',
'implementations.implementation_list': (
previous_as_str + ', testy: socorro.uTYPO.middleware'
)
@@ -355,7 +355,7 @@ def test_overriding_implementation_class(self):
self.assertRaises(ImportError, app.main)
config_manager = self._setup_config_manager({
- 'implementations.service_overrides': 'Crash: testy',
+ 'implementations.service_overrides': 'CrashData: fs, Crash: testy',
'implementations.implementation_list': (
previous_as_str + ', testy: socorro.unittest.middleware'
)
View
2  webapp-php/application/models/report.php
@@ -107,7 +107,7 @@ public function getByUUID($uuid, $crash_uri)
}
public function getRawJson($uuid){
- $uri = Kohana::config('webserviceclient.socorro_hostname').'/crash/meta/by/uuid/'.rawurlencode($uuid);
+ $uri = Kohana::config('webserviceclient.socorro_hostname').'/crash_data/datatype/meta/uuid/'.rawurlencode($uuid);
$result = $this->service->get($uri);
return $result;
}
Please sign in to comment.
Something went wrong with that request. Please try again.