Skip to content
Permalink
Browse files
Updated documentation, specifically Distill's restful api.
  • Loading branch information
mooshu1x2 committed Jun 21, 2016
1 parent a746d04 commit 147fab284ba16beb37d6f78173401e7808b93d23
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 138 deletions.
@@ -1,12 +1,6 @@
'''
distill: This package contains a flask app RESTful api for distill
This flask app exposes some restful api endpoints for querying User-ALE.
Very similar to Lucene syntax for basic query operations.
Copyright 2016, The Charles Stark Draper Laboratory
Licensed under Apache Software License.
'''
__license__ = "Apache-2.0"
__revision__ = " $Id: $ "
__docformat__ = 'reStructuredText'

from flask import Flask, request, jsonify
from distill import app
@@ -19,114 +13,150 @@
@app.route ('/', methods=['GET'])
def index ():
"""
curl -XGET https://[hostname]:[port]
Example:
curl -XGET https://[hostname]:[port]
Show Distill version information, connection status, and all registered applications.
.. code-block:: bash
$ curl -XGET https://localhost:8000
{
"author" : "Michelle Beard",
"email" : "mbeard@draper.com",
"name": "Distill",
"status" : true,
"version" : "1.0",
"apps" : {
"xdata_v3" : {
testing: 205,
parsed: 500,
},
"test_app" : {
logs: 500,
parsed: 100,
}
}
}
:return: Distill's status information as JSON blob
"""
return jsonify (name="Distill", version="1.0 alpha", author="Michelle Beard", email="mbeard@draper.com", status=UserAle.getStatus (), applications=UserAle.getApps ())

"""
curl -XPOST https://[hostname]:[port]/create/app_name
curl -XPUT https://[hostname]:[port]/create/app_name
Example:
curl -XPOST https://[hostname]:[port]/xdata_v3
curl -XPUT https://[hostname]:[port]/xdata_v3
Creates an index in Elasticsearch to store user logs to
"""
@app.route ('/create/<app_id>', methods=['POST', 'PUT'])
def create (app_id):
return UserAle.create (app_id)
"""
Registers an application in Distill.
"""
curl -XGET https://[hostname]:[port]/status/app_name
.. code-block:: bash
Example:
curl -XGET https://[hostname]:[port]/status/xdata_v3
$ curl -XPOST https://localhost:8000/xdata_v3
:param app_id: Application name
:return: Newly created application's status as JSON blob
"""
return UserAle.create (app_id)

Presents meta information about index app_name: field names and document types
"""
@app.route ('/status/<app_id>', methods=['GET'])
def status (app_id):
return UserAle.read (app_id)
"""
Presents meta information about an registered application, including field names and document types.
"""
curl -XPOST https://[hostname]:[port]/update/app_name?name="new_app_name"
curl -XPUT https://[hostname]:[port]/update/app_name?name="new_app_name"
.. code-block:: bash
Example:
curl -XPOST https://[hostname]:[port]/update/xdata_v3?name="xdata_v4"
curl -XPUT https://[hostname]:[port]/update/xdata_v3?name="xdata_v4"
$ curl -XGET https://localhost:8000/status/xdata_v3
:param app_id: Application name
:return: Registered applications meta data as JSON blob
"""
return UserAle.read (app_id)

Renames a specific index in Elasticsearch
"""
@app.route ('/update/<app_id>', methods=['POST', 'PUT'])
def update (app_id):
return UserAle.update (app_id)
"""
Renames a specific application
.. code-block:: bash
"""
curl -XDELETE https://[hostname]:[port]/app_name
$ curl -XPOST https://localhost:8000/update/xdata_v3?name="xdata_v4"
Example:
curl -XDELETE https://[hostname]:[port]/xdata_v3
:param app_id: Application name
:return: Boolean response message as JSON blob
"""
return UserAle.update (app_id)

Deletes an index permentantly from Elasticsearch
"""
@app.route ('/delete/<app_id>', methods=['DELETE'])
def delete (app_id):
return UserAle.delete (app_id)
"""
Deletes an application permentantly from Distill
"""
curl -XGET https://[hostname]:[port]/app_name/select?q=*:*&size=100&scroll=true&fl=param1,param2
.. code-block:: bash
Example:
curl -XGET https://[hostname]:[port]/app_name/select?q=session_id:A1234&size=100&scroll=false&fl=param1,param2
$ curl -XDELETE https://localhost:8000/xdata_v3
:param app_id: Application name
:return: Boolean response message as JSON blob
"""
return UserAle.delete (app_id)

Get all data associated with an application
"""
@app.route ('/search/<app_id>', defaults={"app_type" : None}, methods=['GET'])
@app.route ('/search/<app_id>/<app_type>', methods=['GET'])
def search (app_id, app_type):
"""
Search against an application on various fields.
.. code-block:: bash
$ curl -XGET https://[hostname]:[port]/app_name/select?q=session_id:A1234&size=100&scroll=false&fl=param1,param2
:param app_id: Application name
:param app_type: Optional document type to filter against
:param q: Main search query
:param size: Maximum number of documents to return in request
:param scroll: Scroll id if the number of documents exceeds 10,000
:param fl: List of fields to restrict the result set
:return: JSON blob of result set
"""

q = request.args
try:
validate_request (q)
return UserAle.select (app_id, app_type=app_type, params=q)
except ValidationError as e:
return jsonify (error=e.message)

"""
This can be folded into /search api
curl -XGET https://[hostname]:[port]/app_name/stat?elem=button&event=param1,param2
Example:
curl -XGET https://[hostname]:[port]/xdata_v3/testing/?elem=signup&event=click
How many users clicked on my sign up button?
"""
@app.route ('/stat/<app_id>', defaults={"app_type" : None}, methods=['GET'])
@app.route ('/stat/<app_id>/<app_type>', methods=['GET'])
def stat (app_id, app_type):
q = request.args
"""
.. warning:: Not implemented/available
return jsonify (error='Not implemented')
Generic histogram counts for a single registered application filtered optionally by document type.
"""
curl -XGET https://[hostname]:[port]/denoise/app_name?save=true&type=parsed
.. code-block:: bash
Example:
curl -XGET https://[hostname]:[port]/denoise/xdata_v3?save=true&type=parsed
$ curl -XGET https://localhost:8000/xdata_v3/testing/?elem=signup&event=click
:param app_id: Application name
:param app_type: Application type
:return: JSON blob of result set
"""
q = request.args
return jsonify (error='Not implemented')

Bootstrap script to cleanup the raw logs. A document type called "parsed"
will be stored with new log created unless specified in the request. Have option to save
parsed results back to data store. These parsed logs can be intergrated with STOUT results
by running the stout bootstrap script
"""
@app.route ('/denoise/<app_id>', methods=['GET'])
def denoise (app_id):
"""
Bootstrap script to cleanup the raw logs. A document type called "parsed"
will be stored with new log created unless specified in the request. Have option to save
parsed results back to data store. These parsed logs can be intergrated with STOUT results
by running the stout bootstrap script.
.. code-block:: bash
$ curl -XGET https://localhost:8000/denoise/xdata_v3?save=true&type=parsed
:param app_id: Application name
:return: JSON blob of status
"""
doc_type = 'parsed'
save = False
q = request.args
@@ -137,32 +167,29 @@ def denoise (app_id):
doc_type = q.get ('type')
return UserAle.denoise (app_id, doc_type=doc_type, save=save)

"""
curl -XGET https://[hostname]:[port]/stout/app_name
curl -XGET https://[hostname]:[port]/stout/app_name/app_type
Example:
curl -XGET https://[hostname]:[port]/stout/xdata_v3
curl -XGET https://[hostname]:[port]/stout/xdata_v3/testing
Bootstrap script to aggregate user ale logs to stout master answer table
This will save the merged results back to ES instance at new index stout
OR denoise data first, then merge with the stout index...
If STOUT is enabled, the select method expects a stout index to exist or otherwise
it will return an error message.
"""
#@app.route ('/stout/<app_id>', defaults={"app_type" : None})
#@app.route ('/stout/<app_id>/<app_type>', methods=['GET'])
@app.route ('/stout', methods=['GET'])
def merge_stout ():
"""
Bootstrap script to aggregate user ale logs to stout master answer table
This will save the merged results back to ES instance at new index stout
OR denoise data first, then merge with the stout index...
If STOUT is enabled, the select method expects a stout index to exist or otherwise
it will return an error message.
.. code-block:: bash
$ curl -XGET https://locahost:8000/stout/xdata_v3
:return: Status message
"""
flag = app.config ['ENABLE_STOUT']
if flag:
return Stout.ingest ()
return jsonify (status="STOUT is disabled.")

"""
Generic Error Message
"""
@app.errorhandler(404)
def page_not_found (error):
"""
Generic Error Message
"""
return "Unable to find Distill."
@@ -1,12 +1,6 @@
'''
distill: This package contains a flask app RESTful api for distill
This flask app exposes some restful api endpoints for querying User-ALE.
Very similar to Lucene syntax for basic query operations.
Copyright 2016, The Charles Stark Draper Laboratory
Licensed under Apache Software License.
'''
__license__ = "Apache License 2.0"
__revision__ = " $Id: $ "
__docformat__ = 'reStructuredText'

from elasticsearch_dsl import DocType, String, Boolean, Date, Float, Search
from elasticsearch_dsl.query import MultiMatch, Match, Q
@@ -25,35 +19,17 @@
import yaml
import urllib2

"""
Generic class supporting basic CRUD operations
"""
class UserAle (object):

"""
Get Status of Elasticsearch Instance, show all applications registered and the number
of logs in each application (based on doc_type).
Example:
{
"author" : "Michelle Beard",
"email" : "mbeard@draper.com",
"name": "Distill",
"status" : true,
"version" : "1.0",
"apps" : {
"xdata_v3" : {
testing: 205,
parsed: 500,
},
"test_app" : {
logs: 500,
parsed: 100,
}
}
}
""" Generic class supporting basic CRUD operations
"""

@staticmethod
def getStatus ():
""" Fetch the status of an Elasticsearch instance.
:return: True/False if connection to Elasticsearch instance has been established.
:rtype: bool
"""
try:
res = es.ping ()
except ConnectionError as e:
@@ -62,6 +38,13 @@ def getStatus ():

@staticmethod
def getApps ():
""" Fetch all the registered applications for an Elasticsearch instance.
.. note:: Privated indexes starting with a period are not included in the result set.
:return: A
:rtype: dict
"""
doc = {}
try:
cluster_status = es.cat.indices (h=["index"], pri=False)

0 comments on commit 147fab2

Please sign in to comment.