WIP! Availability API

 * ES queries are far from perfect
 * missed unit tests

availability/api/v1/api.py

@@ -0,0 +1,158 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+import flask
+
+from availability import config
+from availability import storage
+
+
+LOG = logging.getLogger("api")
+LOG.setLevel(config.get_config().get("logging", {}).get("level", "INFO"))
+
+# NOTE(maretskiy): Cache names and services for validation and performance
+REGIONS = {r["name"]: r["services"]
+           for r in config.get_config().get("regions")}
+
+PERIODS = {
+    "day": ("now-1d", "1h"),
+    "week": ("now-1w", "1d"),
+    "month": ("now-1M", "1d")
+}
+
+
+def process_buckets(buckets, round_to=3):
+    """Convert ES buckets into API format, calculate average.
+
+    :param buckets: ES buckets list
+    :param round_to: int round depth
+    :returns: {"availability_data": <list>, "availability": <float>}
+    """
+    data = []
+    status_sum = 0
+
+    for b in buckets:
+        status = b["status"]["value"] or 0
+        status = round(status, round_to)
+        status_sum += status
+        data.append([b["key_as_string"], status])
+
+    # TODO(maretskiy): get AVG from ES, do not calculate it here
+    availability = round((status_sum / len(data)), round_to)
+    return {"availability_data": data,
+            "availability": availability}
+
+
+def query_region(region, period):
+    gte, interval = PERIODS[period]
+    query = {
+        "size": 0,
+        "query": {"range": {"time": {"lte": "now",
+                                     "gte": gte,
+                                     "format": "date_optional_time"}}},
+        "aggs": {
+            "availability": {
+                "date_histogram": {"field": "time",
+                                   "interval": interval,
+                                   "format": "yyyy-MM-dd'T'HH:mm",
+                                   "min_doc_count": 0},
+                "aggs": {"status": {"avg": {"field": "status"}}}
+            }
+        }
+    }
+    result = storage.es_search(region, body=query)
+    if not result:
+        return None
+    buckets = result["aggregations"]["availability"]["buckets"]
+    return process_buckets(buckets)
+
+
+def query_region_by_services(region, period):
+    gte, interval = PERIODS[period]
+    services_av = {}
+    for service in REGIONS[region]:
+        service_name = service["name"]
+        query = {
+            "size": 0,
+            "query": {
+                "bool": {
+                    "must": [
+                        {
+                            "range": {
+                                "time": {"lte": "now",
+                                         "gte": gte,
+                                         "format": "date_optional_time"}}
+                        },
+                        {
+                            "term": {"name": service_name}
+                        }
+                    ]
+                }
+            },
+            "aggs": {
+                "availability": {
+                    "date_histogram": {"field": "time",
+                                       "interval": interval,
+                                       "format": "yyyy-MM-dd'T'HH:mm",
+                                       "min_doc_count": 0},
+                    "aggs": {"status": {"avg": {"field": "status"}}}
+                }
+            }
+        }
+        result = storage.es_search(region, body=query)
+        if not result:
+            # NOTE(maretskiy): Skip if some issues with Elastic. Ideas?
+            continue
+        buckets = result["aggregations"]["availability"]["buckets"]
+        services_av[service_name] = process_buckets(buckets)
+    return services_av
+
+
+bp = flask.Blueprint("availability", __name__)
+
+
+@bp.route("/availability/<period>")
+def get_availability(period):
+    if period not in PERIODS:
+        return flask.jsonify({"error": "Not found"}), 404
+
+    regions_av = {}
+    for region in REGIONS:
+        availability = query_region(region, period)
+        if not availability:
+            # NOTE(maretskiy): Skip if some issues with Elastic. Ideas?
+            continue
+        regions_av[region] = availability
+
+    return flask.jsonify({"availability": regions_av, "period": period})
+
+
+@bp.route("/region/<region>/availability/<period>")
+def get_region_availability(region, period):
+    if (period not in PERIODS) or (region not in REGIONS):
+        return flask.jsonify({"error": "Not found"}), 404
+
+    services_av = query_region_by_services(region, period)
+    result = {"availability": services_av,
+              "region": region,
+              "period": period}
+
+    return flask.jsonify(result)
+
+
+def get_blueprints():
+    return [["", bp]]

availability/main.py

@@ -16,21 +16,26 @@
 import flask
 from flask_helpers import routing
 
+from availability.api.v1 import api
 from availability import config
 
 
 app = flask.Flask(__name__, static_folder=None)
 app.config.update(config.get_config()["flask"])
 
 
-app = routing.add_routing_map(app, html_uri=None, json_uri="/")
-
-
 @app.errorhandler(404)
 def not_found(error):
     return flask.jsonify({"error": "Not Found"}), 404
 
 
+for url_prefix, blueprint in api.get_blueprints():
+    app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix)
+
+
+app = routing.add_routing_map(app, html_uri=None, json_uri="/")
+
+
 def main():
     app.run(host=app.config.get("HOST", "0.0.0.0"),
             port=app.config.get("PORT", 5000))

availability/storage.py

@@ -82,3 +82,21 @@ def ensure_es_index_exists(index):
             "Something went wrong with Elasticsearch: %s" % str(e))
         return None
     return es
+
+
+def es_search(region, body):
+    """Search availability by region.
+
+    :param region: str region name
+    :param body: dict ES query
+    :returns: dict search results
+    """
+    es = get_elasticsearch()
+    try:
+        index = "ms_availability_%s" % region
+        return es.search(index=index, doc_type="service_availability",
+                         body=body)
+    except elasticsearch.exceptions.ElasticsearchException as e:
+        LOG.error("Search query has failed:\nIndex: %s\nBody: %s\nError: %s"
+                  % (index, body, str(e)))
+        return None

tests/unit/test_main.py

@@ -40,4 +40,7 @@ def test_main(self, mock_app):
     def test_api_map(self):
         code, resp = self.get("/")
         self.assertEqual(200, code)
-        self.assertEqual([], resp)
+        self.assertEqual(2, len(resp))
+        self.assertIn({"endpoint": u"availability.get_availability",
+                       "methods": ["GET", "HEAD", "OPTIONS"],
+                       "uri": "/api/v1/availability/<period>"}, resp)