Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

enure throttle level is always set fixes 731325

  • Loading branch information...
commit c8bb37623e2f697411a7126b9253ed3a6cf064e0 1 parent a978f8c
@schalkneethling schalkneethling authored
View
144 docs/addaservice.rst
@@ -189,3 +189,147 @@ use it in the WebApp. If so, have a look at :ref:`ui-chapter`.
You might also want to document it. We are keeping track of all existing
services' documentation in our :ref:`middleware-chapter` page. Please add
yours!
+
+Writing a PostgreSQL middleware unit test
+-----------------------------------------
+
+First create your new test file in the appropriate localtion as specified above,
+for example socorro/unittest/external/postgresql/test_myservice.py
+
+Next you want to import the following:
+::
+ from socorro.external.postgresql.myservice import MyService
+ import socorro.unittest.testlib.util as testutil
+
+As this is a PostgreSQL service unit test we also add:
+::
+ from .unittestbase import PostgreSQLTestCase
+
+Next item to add is your setup_module function, below is a barebones version that
+would be sufficient for most tests:
+::
+ #------------------------------------------------------------------------------
+ def setup_module():
+ testutil.nosePrintModule(__file__)
+
+Next is the setup function in which you create and populate your dummy table(s)
+::
+ #==============================================================================
+ class TestMyService(PostgreSQLTestCase):
+
+ #--------------------------------------------------------------------------
+ def setUp(self):
+
+ super(TestMyService, self).setUp()
+
+ cursor = self.connection.cursor()
+
+ #Create table
+ cursor.execute("""
+ CREATE TABLE product_info
+ (
+ product_version_id integer not null,
+ product_name citext,
+ version_string citext,
+ );
+ """)
+
+ # Insert data
+ cursor.execute("""
+ INSERT INTO product_info VALUES
+ (
+ 1,
+ '%s',
+ '%s'
+ );
+ """ % ("Firefox", "8.0"))
+
+ self.connection.commit()
+
+For your test table(s) you can include as many, or as few, columns and rows of data as your tests
+will require. Next we add the tearDown function that will clean up after our tests has run, by
+dropping tables we created in the setUp function.
+::
+ #--------------------------------------------------------------------------
+ def tearDown(self):
+ """ Cleanup the database, delete tables and functions """
+ cursor = self.connection.cursor()
+ cursor.execute("""
+ DROP TABLE product_info;
+ """)
+ self.connection.commit()
+ super(TestProducts, self).tearDown()
+
+Next, we write our actual tests against the dummy data we created in setUp. First step is to create an
+instance of the class we are going to test:
+::
+ #--------------------------------------------------------------------------
+ def test_get(self):
+ products = Products(config=self.config)
+
+Next we write our first test passing the parameters to our function it expects:
+::
+ #......................................................................
+ # Test 1: find one exact match for one product and one version
+ params = {
+ "versions": "Firefox:8.0"
+ }
+
+Next we call our function passing the above parameters:
+::
+ res = products.get_versions(**params)
+
+The above will now return a response that we need to test and determine whether it contains what we expect.
+In order to do this we create our expected response:
+::
+ res_expected = {
+ "hits": [
+ {
+ "product_version_id": 1,
+ "product_name": "Firefox",
+ "version_string": "8.0"
+ }
+ ],
+ "total": 1
+ }
+
+And finally we call the assertEquals function to test whether our response matches our expected response:
+::
+ self.assertEqual(res, res_expected)
+
+Running a PostgreSQL middleware unit test
+-----------------------------------------
+
+If you have not already done so, install nose tests. From the commons line run the command:
+::
+ sudo apt-get install python-nose
+
+Once the installation completes change directory to, socorro/unittest/config/ and run the following:
+::
+ cp commonconfig.py.dist commonconfig.py
+
+Now you can open up the file and edit it's contents to match your testing environment. If you are running this in a VM via
+Socorro Vagrant, you can leave the content of the file as is. Next cd into socorro/unittest. To run all of the unit tests,
+run the following:
+::
+ nosetests
+
+When writing a new test you most likely are more interested in running your own, and just your own, instead of running all
+of the unit tests that form part of Socorro. If your test is located in, for example unittest/external/postgresql/test_myservice.py
+then you can run your test as follows:
+::
+ nosetests socorro.external.postgresql.test_myservice
+
+Ensuring good style
+-------------------
+
+To ensure that the Python code you wrote passes PEP8 you need to run check.py.
+To do this your first step is to install it. From the terminal run:
+::
+ pip install -e git://github.com/jbalogh/check.git#egg=check
+
+P.S. You may need to sudo the command above
+
+Once installed, run the following:
+::
+ check.py /path/to/your/file
View
50 docs/middleware.rst
@@ -18,6 +18,7 @@ New-style, documented services
* `extensions/ <#id7>`_
* products/
* `products/builds/ <#products-builds>`_
+ * `products/versions/ <#products-versions>`_
* report/
* `report/list/ <#list-report>`_
* search/
@@ -409,6 +410,55 @@ Return an array of objects::
...
]
+Products Versions
+-----------------
+
+Return information about product(s) and version(s) passed to the service.
+
+API specifications
+^^^^^^^^^^^^^^^^^^
+
++----------------+--------------------------------------------------------------------------------+
+| HTTP method | GET |
++----------------+--------------------------------------------------------------------------------+
+| URL schema | /products/(.*) |
++----------------+--------------------------------------------------------------------------------+
+| Full URL | /products/versions/(versions)/ |
++----------------+--------------------------------------------------------------------------------+
+| Example | http://socorro-api/bpapi/products/versions/Firefox:9.0a1/ |
++----------------+--------------------------------------------------------------------------------+
+
+Mandatory parameters
+^^^^^^^^^^^^^^^^^^^^
+
++----------+---------------------------+---------------+----------------------------------------+
+| Name | Type of value | Default value | Description |
++==========+===========================+===============+========================================+
+| versions | String or list of strings | None | Several product:version strings can |
+| | | | be specified, separated by a + symbol. |
++----------+---------------------------+---------------+----------------------------------------+
+
+Return value
+^^^^^^^^^^^^
+
+Return an object with an array of results labeled as hits and a total::
+
+ {
+ "hits": [
+ {
+ "is_featured": boolean,
+ "throttle": float,
+ "end_date": "string",
+ "start_date": "integer",
+ "build_type": "string",
+ "product": "string",
+ "version": "string"
+ }
+ ...
+ ],
+ "total": 1
+ }
+
.. ############################################################################
Search API
############################################################################
View
2  scripts/config/webapiconfig.py.dist
@@ -126,6 +126,7 @@ import socorro.middleware.crashes_comments_service as crashes_comments
import socorro.middleware.crash_service as crash_new
import socorro.middleware.extensions_service as extensions
import socorro.middleware.crashes_paireduuid_service as crashes_paireduuid
+import socorro.middleware.products_versions_service as products_versions
servicesList = cm.Option()
servicesList.doc = 'a python list of classes to offer as services'
@@ -154,6 +155,7 @@ servicesList.default = [
crash_new.Crash,
extensions.Extensions,
crashes_paireduuid.CrashesPaireduuid,
+ products_versions.ProductsVersions,
]
crashBaseUrl = cm.Option()
View
97 socorro/external/postgresql/products.py
@@ -0,0 +1,97 @@
+import logging
+import psycopg2
+
+from socorro.external.postgresql.base import add_param_to_dict, PostgreSQLBase
+from socorro.lib import datetimeutil, external_common
+
+import socorro.database.database as db
+
+logger = logging.getLogger("webapi")
+
+
+class MissingOrBadArgumentException(Exception):
+ pass
+
+
+class Products(PostgreSQLBase):
+
+ def get_versions(self, **kwargs):
+ """ Return product information for one or more product:version
+ combinations """
+ filters = [
+ ("versions", None, ["list", "str"])
+ ]
+
+ params = external_common.parse_arguments(filters, kwargs)
+
+ if not params.versions or params.versions[0] == '':
+ raise MissingOrBadArgumentException(
+ "Mandatory parameter 'versions' missing or empty")
+
+ products = []
+ (params["products_versions"],
+ products) = self.parse_versions(params["versions"], [])
+
+ sql_select = """
+ SELECT product_name as product,
+ version_string as version,
+ start_date,
+ end_date,
+ is_featured,
+ build_type,
+ throttle::float
+ FROM product_info WHERE
+ """
+
+ sql_where = []
+ versions_list = []
+ products_list = []
+ for x in range(0, len(params["products_versions"]), 2):
+ products_list.append(params["products_versions"][x])
+ versions_list.append(params["products_versions"][x + 1])
+
+ sql_where = ["(product_name = %(product" + str(x) +
+ ")s AND version_string = %(version" + str(x) + ")s)"
+ for x in range(len(products_list))]
+
+ sql_params = {}
+ sql_params = add_param_to_dict(sql_params, "product", products_list)
+ sql_params = add_param_to_dict(sql_params, "version", versions_list)
+
+ sql_query = " ".join(
+ ("/* socorro.external.postgresql.Products.get_versions */",
+ sql_select, " OR ".join(sql_where)))
+
+ json_result = {
+ "total": 0,
+ "hits": []
+ }
+
+ try:
+ connection = self.database.connection()
+ cur = connection.cursor()
+ results = db.execute(cur, sql_query, sql_params)
+ except psycopg2.Error:
+ logger.error(
+ "Failed retrieving products_versions data from PostgreSQL",
+ exc_info=True)
+ else:
+ for product in results:
+ row = dict(zip((
+ "product",
+ "version",
+ "start_date",
+ "end_date",
+ "is_featured",
+ "build_type",
+ "throttle"), product))
+ json_result["hits"].append(row)
+ row["start_date"] = datetimeutil.date_to_string(
+ row["start_date"])
+ row["end_date"] = datetimeutil.date_to_string(
+ row["end_date"])
+ json_result["total"] = len(json_result["hits"])
+
+ return json_result
+ finally:
+ connection.close()
View
24 socorro/middleware/products_versions_service.py
@@ -0,0 +1,24 @@
+import logging
+
+from socorro.middleware.service import DataAPIService
+
+logger = logging.getLogger("webapi")
+
+
+class ProductsVersions(DataAPIService):
+
+ service_name = "products"
+ uri = "/products/(.*)"
+
+ def __init__(self, config):
+ super(ProductsVersions, self).__init__(config)
+ logger.debug('ProductsVersions service __init__')
+
+ def get(self, *args):
+ params = self.parse_query_string(args[0])
+
+ module = self.get_module(params)
+
+ impl = module.Products(config=self.context)
+
+ return impl.get_versions(**params)
View
168 socorro/unittest/external/postgresql/test_products.py
@@ -0,0 +1,168 @@
+import datetime
+
+from socorro.external.postgresql.products import Products
+from socorro.external.postgresql.products import MissingOrBadArgumentException
+from socorro.lib import datetimeutil
+import socorro.unittest.testlib.util as testutil
+
+from .unittestbase import PostgreSQLTestCase
+
+#------------------------------------------------------------------------------
+def setup_module():
+ testutil.nosePrintModule(__file__)
+
+#==============================================================================
+class TestProducts(PostgreSQLTestCase):
+ """Test socorro.external.postgresql.products.Products class. """
+
+ #--------------------------------------------------------------------------
+ def setUp(self):
+ """ Populate product_info table with fake data """
+ super(TestProducts, self).setUp()
+
+ cursor = self.connection.cursor()
+
+ #Create table
+ cursor.execute("""
+ CREATE TABLE product_info
+ (
+ product_name citext,
+ version_string citext,
+ start_date timestamp without time zone,
+ end_date timestamp without time zone,
+ is_featured boolean,
+ build_type citext,
+ throttle numeric(5,2)
+ );
+ """)
+
+ # Insert data
+ now = datetimeutil.utc_now().date()
+ cursor.execute("""
+ INSERT INTO product_info VALUES
+ (
+ 'Firefox',
+ '8.0',
+ '%s',
+ '%s',
+ False,
+ 'Release',
+ 10.00
+ ),
+ (
+ 'Firefox',
+ '11.0.1',
+ '%s',
+ '%s',
+ False
+ 'Release',
+ 20.00
+ ),
+ (
+ 'Thunderbird',
+ '10.0.2b',
+ '%s',
+ '%s',
+ False,
+ 'Release',
+ 30.00
+ );
+ """ % (now, now,
+ now, now,
+ now, now))
+
+ self.connection.commit()
+
+ #--------------------------------------------------------------------------
+ def tearDown(self):
+ """ Cleanup the database, delete tables and functions """
+ cursor = self.connection.cursor()
+ cursor.execute("""
+ DROP TABLE product_info;
+ """)
+ self.connection.commit()
+ super(TestProducts, self).tearDown()
+
+ #--------------------------------------------------------------------------
+ def test_get(self):
+ products = Products(config=self.config)
+ now = datetimeutil.utc_now()
+ now = datetime.datetime(now.year, now.month, now.day, tzinfo=now.tzinfo)
+ now_str = datetimeutil.date_to_string(now)
+
+ #......................................................................
+ # Test 1: find one exact match for one product and one version
+ params = {
+ "versions": "Firefox:8.0"
+ }
+ res = products.get_versions(**params)
+ res_expected = {
+ "hits": [
+ {
+ "product": "Firefox",
+ "version": "8.0",
+ "start_date": now_str,
+ "end_date": now_str,
+ "is_featured": False,
+ "build_type": "Release",
+ "throttle": 10.0
+ }
+ ],
+ "total": 1
+ }
+
+ self.assertEqual(res, res_expected)
+
+ #......................................................................
+ # Test 2: Find two different products with their correct verions
+ params = {
+ "versions": ["Firefox:11.0.1", "Thunderbird:10.0.2b"]
+ }
+ res = products.get_versions(**params)
+ res_expected = {
+ "hits": [
+ {
+ "product": "Firefox",
+ "version": "11.0.1",
+ "start_date": now_str,
+ "end_date": now_str,
+ "is_featured": False,
+ "build_type": "Release",
+ "throttle": 20.0
+ },
+ {
+ "product": "Thunderbird",
+ "version": "10.0.2b",
+ "start_date": now_str,
+ "end_date": now_str,
+ "is_featured": False,
+ "build_type": "Release",
+ "throttle": 30.0
+ }
+ ],
+ "total": 2
+ }
+
+ self.assertEqual(res, res_expected)
+
+ #......................................................................
+ # Test 3: empty result, no products:version found
+ params = {
+ "versions": "Firefox:14.0"
+ }
+ res = products.get_versions(**params)
+ res_expected = {
+ "hits": [],
+ "total": 0
+ }
+
+ self.assertEqual(res, res_expected)
+
+ #......................................................................
+ # Test 4: exception when no parameter is passed
+ params = {
+ "versions": ['']
+ }
+ self.assertRaises(MissingOrBadArgumentException,
+ products.get_versions,
+ **params)
View
8 webapp-php/application/controllers/daily.php
@@ -155,6 +155,14 @@ public function index() {
$throttle[] = $featured_version->throttle;
}
}
+ // if a version is provided but no throttle level,
+ // we need to call the products_versions service to get the throttle level
+ if(empty($throttle) && isset($versions)) {
+ $response = $this->model->getVersionInfo($product, $versions);
+ foreach($response->hits as $version) {
+ $throttle[] = $version->throttle;
+ }
+ }
if (isset($parameters['form_selection']) && $parameters['form_selection'] == 'by_report_type') {
$url_csv = $this->_commonCsvURL($product, $versions, $operating_systems, $date_start, $date_end, $form_selection, $throttle);
View
19 webapp-php/application/models/daily.php
@@ -552,6 +552,25 @@ public function get($product, $versions, $operating_systems, $start_date, $end_d
}
return false;
}
+
+ /**
+ * Get version information that includes throttle level from products_versions service
+ *
+ * @access public
+ * @param string The product
+ * @param string The version number
+ * @return object The JSON result
+ */
+ public function getVersionInfo($product, $versions) {
+ $host = Kohana::config('webserviceclient.socorro_hostname');
+ $p = rawurlencode($product);
+ $v = $this->encodeArray($versions);
+ $url = $host . "/products/versions/" . $p . ":" . $v;
+
+ $lifetime = Kohana::config('webserviceclient.topcrash_vers_rank_cache_minutes', 60) * 60;
+ $response = $this->service->get($url, 'json', $lifetime);
+ return $response;
+ }
/**
* Fetch records for active daily users / installs by crash report type
Please sign in to comment.
Something went wrong with that request. Please try again.