From a05356ebfe0fe462f20143625ec8c942847348de Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 6 Jan 2016 17:20:01 -0800 Subject: [PATCH 1/3] Remove Flask-Autodoc entirely It's unable to handle decorated functions which we need for login/auth --- digits/dataset/images/classification/views.py | 6 +- digits/dataset/images/generic/views.py | 5 +- digits/dataset/images/views.py | 3 +- digits/dataset/views.py | 3 +- digits/model/images/classification/views.py | 8 +- digits/model/images/generic/views.py | 7 +- digits/model/views.py | 8 +- digits/views.py | 11 +- digits/webapp.py | 12 - docs/API.md | 155 ------ docs/FlaskRoutes.md | 450 ------------------ requirements_test.txt | 1 - scripts/generate_docs.py | 224 --------- scripts/test_generate_docs.py | 59 --- 14 files changed, 8 insertions(+), 944 deletions(-) delete mode 100644 docs/API.md delete mode 100644 docs/FlaskRoutes.md delete mode 100755 scripts/generate_docs.py delete mode 100644 scripts/test_generate_docs.py diff --git a/digits/dataset/images/classification/views.py b/digits/dataset/images/classification/views.py index 18a55b566..b89ef1dda 100644 --- a/digits/dataset/images/classification/views.py +++ b/digits/dataset/images/classification/views.py @@ -7,7 +7,7 @@ from digits import utils from digits.utils.forms import fill_form_if_cloned, save_form_to_job from digits.utils.routing import request_wants_json, job_from_request -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.dataset import tasks from forms import ImageClassificationDatasetForm from job import ImageClassificationDatasetJob @@ -254,7 +254,6 @@ def from_files(job, form): @app.route(NAMESPACE + '/new', methods=['GET']) -@autodoc('datasets') def image_classification_dataset_new(): """ Returns a form for a new ImageClassificationDatasetJob @@ -268,7 +267,6 @@ def image_classification_dataset_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) -@autodoc(['datasets', 'api']) def image_classification_dataset_create(): """ Creates a new ImageClassificationDatasetJob @@ -328,7 +326,6 @@ def show(job): return flask.render_template('datasets/images/classification/show.html', job=job) @app.route(NAMESPACE + '/summary', methods=['GET']) -@autodoc('datasets') def image_classification_dataset_summary(): """ Return a short HTML summary of a DatasetJob @@ -364,7 +361,6 @@ def entries(self): yield item @app.route(NAMESPACE + '/explore', methods=['GET']) -@autodoc('datasets') def image_classification_dataset_explore(): """ Returns a gallery consisting of the images of one of the dbs diff --git a/digits/dataset/images/generic/views.py b/digits/dataset/images/generic/views.py index 82cf0a643..c78c1696a 100644 --- a/digits/dataset/images/generic/views.py +++ b/digits/dataset/images/generic/views.py @@ -4,7 +4,7 @@ from digits.utils.forms import fill_form_if_cloned, save_form_to_job from digits.utils.routing import request_wants_json, job_from_request -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.dataset import tasks from forms import GenericImageDatasetForm from job import GenericImageDatasetJob @@ -12,7 +12,6 @@ NAMESPACE = '/datasets/images/generic' @app.route(NAMESPACE + '/new', methods=['GET']) -@autodoc('datasets') def generic_image_dataset_new(): """ Returns a form for a new GenericImageDatasetJob @@ -26,7 +25,6 @@ def generic_image_dataset_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) -@autodoc(['datasets', 'api']) def generic_image_dataset_create(): """ Creates a new GenericImageDatasetJob @@ -118,7 +116,6 @@ def show(job): return flask.render_template('datasets/images/generic/show.html', job=job) @app.route(NAMESPACE + '/summary', methods=['GET']) -@autodoc('datasets') def generic_image_dataset_summary(): """ Return a short HTML summary of a DatasetJob diff --git a/digits/dataset/images/views.py b/digits/dataset/images/views.py index ead29069f..6edbece9f 100644 --- a/digits/dataset/images/views.py +++ b/digits/dataset/images/views.py @@ -8,14 +8,13 @@ import digits from digits import utils -from digits.webapp import app, autodoc +from digits.webapp import app import classification.views import generic.views NAMESPACE = '/datasets/images' @app.route(NAMESPACE + '/resize-example', methods=['POST']) -@autodoc('datasets') def image_dataset_resize_example(): """ Resizes the example image, and returns it as a string of png data diff --git a/digits/dataset/views.py b/digits/dataset/views.py index 7bc20152a..67b8dfbc4 100644 --- a/digits/dataset/views.py +++ b/digits/dataset/views.py @@ -3,7 +3,7 @@ import flask import werkzeug.exceptions -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.utils.routing import request_wants_json import images.views import images as dataset_images @@ -12,7 +12,6 @@ @app.route(NAMESPACE + '.json', methods=['GET']) @app.route(NAMESPACE + '', methods=['GET']) -@autodoc(['datasets', 'api']) def datasets_show(job_id): """ Show a DatasetJob diff --git a/digits/model/images/classification/views.py b/digits/model/images/classification/views.py index a93d662e7..f16cf78f0 100644 --- a/digits/model/images/classification/views.py +++ b/digits/model/images/classification/views.py @@ -12,7 +12,7 @@ from digits.config import config_value from digits import utils from digits.utils.routing import request_wants_json, job_from_request -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.dataset import ImageClassificationDatasetJob from digits import frameworks from forms import ImageClassificationModelForm @@ -24,7 +24,6 @@ NAMESPACE = '/models/images/classification' @app.route(NAMESPACE + '/new', methods=['GET']) -@autodoc('models') def image_classification_model_new(): """ Return a form for a new ImageClassificationModelJob @@ -50,7 +49,6 @@ def image_classification_model_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) -@autodoc(['models', 'api']) def image_classification_model_create(): """ Create a new ImageClassificationModelJob @@ -235,7 +233,6 @@ def show(job): return flask.render_template('models/images/classification/show.html', job=job, framework_ids = [fw.get_id() for fw in frameworks.get_frameworks()]) @app.route(NAMESPACE + '/large_graph', methods=['GET']) -@autodoc('models') def image_classification_model_large_graph(): """ Show the loss/accuracy graph, but bigger @@ -246,7 +243,6 @@ def image_classification_model_large_graph(): @app.route(NAMESPACE + '/classify_one.json', methods=['POST']) @app.route(NAMESPACE + '/classify_one', methods=['POST', 'GET']) -@autodoc(['models', 'api']) def image_classification_model_classify_one(): """ Classify one image and return the top 5 classifications @@ -304,7 +300,6 @@ def image_classification_model_classify_one(): @app.route(NAMESPACE + '/classify_many.json', methods=['POST']) @app.route(NAMESPACE + '/classify_many', methods=['POST', 'GET']) -@autodoc(['models', 'api']) def image_classification_model_classify_many(): """ Classify many images and return the top 5 classifications for each @@ -389,7 +384,6 @@ def image_classification_model_classify_many(): ) @app.route(NAMESPACE + '/top_n', methods=['POST']) -@autodoc('models') def image_classification_model_top_n(): """ Classify many images and show the top N images per category by confidence diff --git a/digits/model/images/generic/views.py b/digits/model/images/generic/views.py index 2777f1072..341ee1ec5 100644 --- a/digits/model/images/generic/views.py +++ b/digits/model/images/generic/views.py @@ -12,7 +12,7 @@ from digits import utils from digits.utils.routing import request_wants_json, job_from_request from digits.utils.forms import fill_form_if_cloned, save_form_to_job -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.dataset import GenericImageDatasetJob from forms import GenericImageModelForm from job import GenericImageModelJob @@ -22,7 +22,6 @@ NAMESPACE = '/models/images/generic' @app.route(NAMESPACE + '/new', methods=['GET']) -@autodoc('models') def generic_image_model_new(): """ Return a form for a new GenericImageModelJob @@ -47,7 +46,6 @@ def generic_image_model_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) -@autodoc(['models', 'api']) def generic_image_model_create(): """ Create a new GenericImageModelJob @@ -218,7 +216,6 @@ def show(job): return flask.render_template('models/images/generic/show.html', job=job) @app.route(NAMESPACE + '/large_graph', methods=['GET']) -@autodoc('models') def generic_image_model_large_graph(): """ Show the loss/accuracy graph, but bigger @@ -229,7 +226,6 @@ def generic_image_model_large_graph(): @app.route(NAMESPACE + '/infer_one.json', methods=['POST']) @app.route(NAMESPACE + '/infer_one', methods=['POST', 'GET']) -@autodoc(['models', 'api']) def generic_image_model_infer_one(): """ Infer one image @@ -280,7 +276,6 @@ def generic_image_model_infer_one(): @app.route(NAMESPACE + '/infer_many.json', methods=['POST']) @app.route(NAMESPACE + '/infer_many', methods=['POST', 'GET']) -@autodoc(['models', 'api']) def generic_image_model_infer_many(): """ Infer many images diff --git a/digits/model/views.py b/digits/model/views.py index 179570977..217d61208 100644 --- a/digits/model/views.py +++ b/digits/model/views.py @@ -12,7 +12,7 @@ import digits -from digits.webapp import app, scheduler, autodoc +from digits.webapp import app, scheduler from digits.utils import time_filters from digits.utils.routing import request_wants_json from . import ModelJob @@ -25,7 +25,6 @@ NAMESPACE = '/models/' @app.route(NAMESPACE, methods=['GET']) -@autodoc(['models']) def models_index(): column_attrs = list(get_column_attrs()) raw_jobs = [j for j in scheduler.jobs.values() if isinstance(j, ModelJob)] @@ -82,7 +81,6 @@ def models_index(): @app.route(NAMESPACE + '.json', methods=['GET']) @app.route(NAMESPACE + '', methods=['GET']) -@autodoc(['models', 'api']) def models_show(job_id): """ Show a ModelJob @@ -106,7 +104,6 @@ def models_show(job_id): 'Invalid job type') @app.route(NAMESPACE + 'customize', methods=['POST']) -@autodoc('models') def models_customize(): """ Returns a customized file for the ModelJob based on completed form fields @@ -147,7 +144,6 @@ def models_customize(): }) @app.route(NAMESPACE + 'visualize-network', methods=['POST']) -@autodoc('models') def models_visualize_network(): """ Returns a visualization of the custom network as a string of PNG data @@ -162,7 +158,6 @@ def models_visualize_network(): return ret @app.route(NAMESPACE + 'visualize-lr', methods=['POST']) -@autodoc('models') def models_visualize_lr(): """ Returns a JSON object of data used to create the learning rate graph @@ -217,7 +212,6 @@ def models_visualize_lr(): defaults={'extension': 'tar.gz'}) @app.route(NAMESPACE + '/download.', methods=['GET', 'POST']) -@autodoc('models') def models_download(job_id, extension): """ Return a tarball of all files required to run the model diff --git a/digits/views.py b/digits/views.py index a968630e6..2f8291c4a 100644 --- a/digits/views.py +++ b/digits/views.py @@ -13,7 +13,7 @@ import digits from . import dataset, model from config import config_value -from webapp import app, socketio, scheduler, autodoc +from webapp import app, socketio, scheduler import dataset.views import model.views from digits.utils import errors @@ -22,7 +22,6 @@ @app.route('/index.json', methods=['GET']) @app.route('/', methods=['GET']) -@autodoc(['home', 'api']) def home(): """ DIGITS home page @@ -103,7 +102,6 @@ def get_job_list(cls, running): ### Jobs routes @app.route('/jobs/', methods=['GET']) -@autodoc('jobs') def show_job(job_id): """ Redirects to the appropriate /datasets/ or /models/ page @@ -120,7 +118,6 @@ def show_job(job_id): raise werkzeug.exceptions.BadRequest('Invalid job type') @app.route('/jobs/', methods=['PUT']) -@autodoc('jobs') def edit_job(job_id): """ Edit a job's name and/or notes @@ -150,7 +147,6 @@ def edit_job(job_id): @app.route('/datasets//status', methods=['GET']) @app.route('/models//status', methods=['GET']) @app.route('/jobs//status', methods=['GET']) -@autodoc('jobs') def job_status(job_id): """ Returns a JSON objecting representing the status of a job @@ -169,7 +165,6 @@ def job_status(job_id): @app.route('/datasets/', methods=['DELETE']) @app.route('/models/', methods=['DELETE']) @app.route('/jobs/', methods=['DELETE']) -@autodoc('jobs') def delete_job(job_id): """ Deletes a job @@ -189,7 +184,6 @@ def delete_job(job_id): @app.route('/datasets//abort', methods=['POST']) @app.route('/models//abort', methods=['POST']) @app.route('/jobs//abort', methods=['POST']) -@autodoc('jobs') def abort_job(job_id): """ Aborts a running job @@ -204,7 +198,6 @@ def abort_job(job_id): raise werkzeug.exceptions.Forbidden('Job not aborted') @app.route('/clone/', methods=['POST', 'GET']) -@autodoc('jobs') def clone_job(clone): """ Clones a job with the id , populating the creation page with data saved in @@ -272,7 +265,6 @@ def handle_error(e): ### File serving @app.route('/files/', methods=['GET']) -@autodoc('util') def serve_file(path): """ Return a file in the jobs directory @@ -286,7 +278,6 @@ def serve_file(path): ### Path Completion @app.route('/autocomplete/path', methods=['GET']) -@autodoc('util') def path_autocomplete(): """ Return a list of paths matching the specified preamble diff --git a/digits/webapp.py b/digits/webapp.py index f9bf52c48..fb55b3e45 100644 --- a/digits/webapp.py +++ b/digits/webapp.py @@ -19,18 +19,6 @@ socketio = SocketIO(app) scheduler = digits.scheduler.Scheduler(config_value('gpu_list'), True) -# Set up flask API documentation, if installed -try: - from flask.ext.autodoc import Autodoc - _doc = Autodoc(app) - autodoc = _doc.doc # decorator -except ImportError: - def autodoc(*args, **kwargs): - def _doc(f): - # noop decorator - return f - return _doc - ### Register filters and views app.jinja_env.globals['server_name'] = config_value('server_name') diff --git a/docs/API.md b/docs/API.md deleted file mode 100644 index ae7a0a6fa..000000000 --- a/docs/API.md +++ /dev/null @@ -1,155 +0,0 @@ -# REST API - -*Generated Nov 20, 2015* - -DIGITS exposes its internal functionality through a REST API. -You can access these endpoints by performing a GET or POST on the route, and a JSON object will be returned. -For example: -```sh -curl localhost/index.json -``` - -For more information about other routes used for the web interface, see [this page](FlaskRoutes.md). - -### `/datasets/.json` - -> Show a DatasetJob - -> - -> Returns JSON when requested: - -> {id, name, directory, status} - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/dataset/views.py`](../digits/dataset/views.py) - -### `/datasets/images/classification.json` - -> Creates a new ImageClassificationDatasetJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/dataset/images/classification/views.py`](../digits/dataset/images/classification/views.py) - -### `/datasets/images/generic.json` - -> Creates a new GenericImageDatasetJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/dataset/images/generic/views.py`](../digits/dataset/images/generic/views.py) - -### `/index.json` - -> DIGITS home page - -> Returns information about each job on the server - -> - -> Returns JSON when requested: - -> { - -> datasets: [{id, name, status},...], - -> models: [{id, name, status},...] - -> } - -Methods: **GET** - -Location: [`digits/views.py`](../digits/views.py) - -### `/models/.json` - -> Show a ModelJob - -> - -> Returns JSON when requested: - -> {id, name, directory, status, snapshots: [epoch,epoch,...]} - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models/images/classification.json` - -> Create a new ImageClassificationModelJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/classify_many.json` - -> Classify many images and return the top 5 classifications for each - -> - -> Returns JSON when requested: {classifications: {filename: [[category,confidence],...],...}} - -Methods: **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/classify_one.json` - -> Classify one image and return the top 5 classifications - -> - -> Returns JSON when requested: {predictions: {category: confidence,...}} - -Methods: **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/generic.json` - -> Create a new GenericImageModelJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/infer_many.json` - -> Infer many images - -Methods: **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/infer_one.json` - -> Infer one image - -Methods: **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - diff --git a/docs/FlaskRoutes.md b/docs/FlaskRoutes.md deleted file mode 100644 index dcb309b84..000000000 --- a/docs/FlaskRoutes.md +++ /dev/null @@ -1,450 +0,0 @@ -# Flask Routes - -*Generated Nov 20, 2015* - -Documentation on the various routes used internally for the web application. - -These are all technically RESTful, but they return HTML pages. To get JSON responses, see [this page](API.md). - -### Table of Contents - -* [Home](#home) -* [Jobs](#jobs) -* [Datasets](#datasets) -* [Models](#models) -* [Util](#util) - -## Home - -### `/` - -> DIGITS home page - -> Returns information about each job on the server - -> - -> Returns JSON when requested: - -> { - -> datasets: [{id, name, status},...], - -> models: [{id, name, status},...] - -> } - -Methods: **GET** - -Location: [`digits/views.py`](../digits/views.py) - -## Jobs - -### `/clone/` - -> Clones a job with the id , populating the creation page with data saved in - -Methods: **GET**, **POST** - -Arguments: `clone` - -Location: [`digits/views.py`](../digits/views.py) - -### `/datasets/` - -> Deletes a job - -Methods: **DELETE** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/datasets//abort` - -> Aborts a running job - -Methods: **POST** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/datasets//status` - -> Returns a JSON objecting representing the status of a job - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/jobs/` - -> Redirects to the appropriate /datasets/ or /models/ page - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/jobs/` - -> Edit a job's name and/or notes - -Methods: **PUT** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/jobs/` - -> Deletes a job - -Methods: **DELETE** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/jobs//abort` - -> Aborts a running job - -Methods: **POST** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/jobs//status` - -> Returns a JSON objecting representing the status of a job - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/models/` - -> Deletes a job - -Methods: **DELETE** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/models//abort` - -> Aborts a running job - -Methods: **POST** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -### `/models//status` - -> Returns a JSON objecting representing the status of a job - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/views.py`](../digits/views.py) - -## Datasets - -### `/datasets/` - -> Show a DatasetJob - -> - -> Returns JSON when requested: - -> {id, name, directory, status} - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/dataset/views.py`](../digits/dataset/views.py) - -### `/datasets/images/classification` - -> Creates a new ImageClassificationDatasetJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/dataset/images/classification/views.py`](../digits/dataset/images/classification/views.py) - -### `/datasets/images/classification/explore` - -> Returns a gallery consisting of the images of one of the dbs - -Methods: **GET** - -Location: [`digits/dataset/images/classification/views.py`](../digits/dataset/images/classification/views.py) - -### `/datasets/images/classification/new` - -> Returns a form for a new ImageClassificationDatasetJob - -Methods: **GET** - -Location: [`digits/dataset/images/classification/views.py`](../digits/dataset/images/classification/views.py) - -### `/datasets/images/classification/summary` - -> Return a short HTML summary of a DatasetJob - -Methods: **GET** - -Location: [`digits/dataset/images/classification/views.py`](../digits/dataset/images/classification/views.py) - -### `/datasets/images/generic` - -> Creates a new GenericImageDatasetJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/dataset/images/generic/views.py`](../digits/dataset/images/generic/views.py) - -### `/datasets/images/generic/new` - -> Returns a form for a new GenericImageDatasetJob - -Methods: **GET** - -Location: [`digits/dataset/images/generic/views.py`](../digits/dataset/images/generic/views.py) - -### `/datasets/images/generic/summary` - -> Return a short HTML summary of a DatasetJob - -Methods: **GET** - -Location: [`digits/dataset/images/generic/views.py`](../digits/dataset/images/generic/views.py) - -### `/datasets/images/resize-example` - -> Resizes the example image, and returns it as a string of png data - -Methods: **POST** - -Location: [`digits/dataset/images/views.py`](../digits/dataset/images/views.py) - -## Models - -### `/models/` - -Methods: **GET** - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models/` - -> Show a ModelJob - -> - -> Returns JSON when requested: - -> {id, name, directory, status, snapshots: [epoch,epoch,...]} - -Methods: **GET** - -Arguments: `job_id` - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models//download` - -> Return a tarball of all files required to run the model - -Methods: **GET**, **POST** - -Arguments: `job_id`, `extension` (`tar.gz`) - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models//download.` - -> Return a tarball of all files required to run the model - -Methods: **GET**, **POST** - -Arguments: `job_id`, `extension` - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models/customize` - -> Returns a customized file for the ModelJob based on completed form fields - -Methods: **POST** - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models/images/classification` - -> Create a new ImageClassificationModelJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/classify_many` - -> Classify many images and return the top 5 classifications for each - -> - -> Returns JSON when requested: {classifications: {filename: [[category,confidence],...],...}} - -Methods: **GET**, **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/classify_one` - -> Classify one image and return the top 5 classifications - -> - -> Returns JSON when requested: {predictions: {category: confidence,...}} - -Methods: **GET**, **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/large_graph` - -> Show the loss/accuracy graph, but bigger - -Methods: **GET** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/new` - -> Return a form for a new ImageClassificationModelJob - -Methods: **GET** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/classification/top_n` - -> Classify many images and show the top N images per category by confidence - -Methods: **POST** - -Location: [`digits/model/images/classification/views.py`](../digits/model/images/classification/views.py) - -### `/models/images/generic` - -> Create a new GenericImageModelJob - -> - -> Returns JSON when requested: {job_id,name,status} or {errors:[]} - -Methods: **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/infer_many` - -> Infer many images - -Methods: **GET**, **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/infer_one` - -> Infer one image - -Methods: **GET**, **POST** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/large_graph` - -> Show the loss/accuracy graph, but bigger - -Methods: **GET** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/images/generic/new` - -> Return a form for a new GenericImageModelJob - -Methods: **GET** - -Location: [`digits/model/images/generic/views.py`](../digits/model/images/generic/views.py) - -### `/models/visualize-lr` - -> Returns a JSON object of data used to create the learning rate graph - -Methods: **POST** - -Location: [`digits/model/views.py`](../digits/model/views.py) - -### `/models/visualize-network` - -> Returns a visualization of the custom network as a string of PNG data - -Methods: **POST** - -Location: [`digits/model/views.py`](../digits/model/views.py) - -## Util - -### `/autocomplete/path` - -> Return a list of paths matching the specified preamble - -Methods: **GET** - -Location: [`digits/views.py`](../digits/views.py) - -### `/files/` - -> Return a file in the jobs directory - -> - -> If you install the nginx.site file, nginx will serve files instead - -> and this path will never be used - -Methods: **GET** - -Arguments: `path` - -Location: [`digits/views.py`](../digits/views.py) - diff --git a/requirements_test.txt b/requirements_test.txt index 26418d1c7..6476a1b33 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,7 +1,6 @@ nose>=1.3.1 mock>=1.0.1 beautifulsoup4>=4.4.0 -Flask-Autodoc>=0.1.2 selenium>=2.25.0 coverage>=3.7.1 coveralls>=0.5 diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py deleted file mode 100755 index ffd380ade..000000000 --- a/scripts/generate_docs.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. - -import sys -import os.path -import time -from collections import defaultdict - -# Add path for DIGITS package -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import digits.config; digits.config.load_config() -from digits.webapp import app, _doc as doc - -class DocGenerator(object): - """ - Generates markdown for Flask routes - """ - - def __init__(self, autodoc, - include_groups=None, exclude_groups=None): - """ - Arguments: - autodoc -- an Autodoc instance - - Keyword arguments: - include_groups -- a list of groups to print - exclude_groups -- a list of groups not to print - """ - self.autodoc = autodoc - self.include_groups = include_groups - self.exclude_groups = exclude_groups - self._handle = None - - # get list of groups - group_names = defaultdict(int) - for func, groups in self.autodoc.func_groups.iteritems(): - for group in groups: - group_names[group] += 1 - - first_groups = ['home', 'jobs', 'datasets', 'models'] - hidden_groups = ['all'] - other_groups = [g for g in sorted(group_names.keys()) - if g not in first_groups + hidden_groups] - self._groups = first_groups + other_groups - - def generate(self, filename): - """ - Writes the documentation to file - """ - with open(os.path.join( - os.path.dirname(__file__), - filename), 'w') as self._handle: - groups = [] - for group in self._groups: - if (not self.include_groups or group in self.include_groups) and \ - (not self.exclude_groups or group not in self.exclude_groups): - groups.append(group) - - self.print_header() - self._print_toc(groups) - for group in groups: - self._print_group(group, print_header=(len(groups)>1)) - - def w(self, line='', add_newline=True): - """ - Writes a line to file - """ - if add_newline: - line = '%s\n' % line - self._handle.write(line) - - def _print_header(self, header): - """ - Print the document page header - """ - pass - - def timestamp(self): - """ - Returns a string which notes the current time - """ - return time.strftime('*Generated %b %d, %Y*') - - def _print_toc(self, groups=None): - """ - Print the table of contents - """ - if groups is None: - groups = self._groups - - if len(groups) <= 1: - # No sense printing the TOC - return - - self.w('### Table of Contents') - self.w() - for group in groups: - self.w('* [%s](#%s)' % (group.capitalize(), group)) - self.w() - - def _print_group(self, group, print_header=True): - """ - Print a group of routes - """ - routes = self.get_routes(group) - if not routes: - return - - if print_header: - self.w('## %s' % group.capitalize()) - self.w() - - for route in routes: - self._print_route(route) - - def get_routes(self, group): - """ - Get the routes for this group - """ - return self.autodoc.generate(groups=group) - - - def _print_route(self, route): - """ - Print a route - """ - self.w('### `%s`' % route['rule']) - self.w() - if route['docstring']: - for line in route['docstring'].strip().split('\n'): - self.w('> %s' % line.strip()) - self.w() - self.w('Methods: ' + ', '.join(['**%s**' % m.upper() for m in - sorted(route['methods']) if m not in ['HEAD', 'OPTIONS']])) - self.w() - if route['args'] and route['args'] != ['None']: - args = [] - for arg in route['args']: - args.append('`%s`' % arg) - if route['defaults'] and arg in route['defaults']: - args[-1] = '%s (`%s`)' % (args[-1], route['defaults'][arg]) - self.w('Arguments: ' + ', '.join(args)) - self.w() - if 'location' in route and route['location']: - # get location relative to digits root - digits_root = os.path.dirname( - os.path.dirname( - os.path.normpath(digits.__file__) - ) - ) - filename = os.path.normpath(route['location']['filename']) - if filename.startswith(digits_root): - filename = os.path.relpath(filename, digits_root).replace("\\","/") - self.w('Location: [`%s`](%s)' % ( - filename, - os.path.join('..', filename).replace("\\","/"), - )) - self.w() - - -class ApiDocGenerator(DocGenerator): - """ - Generates API.md - """ - - def __init__(self, *args, **kwargs): - super(ApiDocGenerator, self).__init__(include_groups=['api'], *args, **kwargs) - - def print_header(self): - text = """ -# REST API - -%s - -DIGITS exposes its internal functionality through a REST API. -You can access these endpoints by performing a GET or POST on the route, and a JSON object will be returned. -For example: -```sh -curl localhost/index.json -``` - -For more information about other routes used for the web interface, see [this page](FlaskRoutes.md). -""" % self.timestamp() - self.w(text.strip()) - self.w() - - def get_routes(self, group): - for route in self.autodoc.generate(groups=group): - if '.json' in route['rule']: - yield route - - -class FlaskRoutesDocGenerator(DocGenerator): - """ - Generates FlaskRoutes.md - """ - - def __init__(self, *args, **kwargs): - super(FlaskRoutesDocGenerator, self).__init__(exclude_groups=['api'], *args, **kwargs) - - def print_header(self): - text = """ -# Flask Routes - -%s - -Documentation on the various routes used internally for the web application. - -These are all technically RESTful, but they return HTML pages. To get JSON responses, see [this page](API.md). -""" % self.timestamp() - self.w(text.strip()) - self.w() - - def get_routes(self, group): - for route in self.autodoc.generate(groups=group): - if '.json' not in route['rule']: - yield route - - -if __name__ == '__main__': - with app.app_context(): - ApiDocGenerator(doc).generate('../docs/API.md') - FlaskRoutesDocGenerator(doc).generate('../docs/FlaskRoutes.md') - diff --git a/scripts/test_generate_docs.py b/scripts/test_generate_docs.py deleted file mode 100644 index 48ec37171..000000000 --- a/scripts/test_generate_docs.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. - -import os.path -import sys -import tempfile -import itertools -import unittest -import platform - -try: - import flask.ext.autodoc -except ImportError as e: - raise unittest.SkipTest('Flask-Autodoc not installed') - -# Add path for DIGITS package -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import digits.config; digits.config.load_config() -from digits.webapp import app, _doc as doc -from . import generate_docs as _ - -def check_doc_file(generator, doc_filename): - """ - Checks that the output generated by generator matches the contents of doc_filename - """ - # overcome temporary file permission errors - tmp_file_name = tempfile.mkstemp(suffix='.md') - os.close(tmp_file_name[0]) - with open(tmp_file_name[1],'w+') as tmp_file: - generator.generate(tmp_file_name[1]) - tmp_file.seek(0) - with open(doc_filename) as doc_file: - # memory friendly - for doc_line, tmp_line in itertools.izip(doc_file, tmp_file): - doc_line = doc_line.strip() - tmp_line = tmp_line.strip() - if doc_line.startswith('*Generated') and \ - tmp_line.startswith('*Generated'): - # If the date is different, that's not a problem - pass - elif doc_line != tmp_line: - print '(Previous)', doc_line - print '(New) ', tmp_line - raise RuntimeError('%s needs to be regenerated. Use scripts/generate_docs.py' % doc_filename) - os.remove(tmp_file_name[1]) - -def test_api_md(): - with app.app_context(): - generator = _.ApiDocGenerator(doc) - check_doc_file(generator, - os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'API.md')) - -def test_flask_routes_md(): - with app.app_context(): - generator = _.FlaskRoutesDocGenerator(doc) - check_doc_file(generator, - os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'FlaskRoutes.md')) - - From 2dbc7e9869434e349bcf2f428ff27fc19b8bce6a Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Thu, 10 Dec 2015 13:46:34 -0800 Subject: [PATCH 2/3] Add login routes and utils.auth module --- digits/templates/layout.html | 21 ++++++++--- digits/templates/login.html | 29 ++++++++++++++ digits/utils/__init__.py | 2 +- digits/utils/auth.py | 73 ++++++++++++++++++++++++++++++++++++ digits/views.py | 47 +++++++++++++++++++++-- digits/webapp.py | 12 ++++++ 6 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 digits/templates/login.html create mode 100644 digits/utils/auth.py diff --git a/digits/templates/layout.html b/digits/templates/layout.html index 302548a18..8531d1905 100644 --- a/digits/templates/layout.html +++ b/digits/templates/layout.html @@ -31,14 +31,25 @@ {% block nav %} {% endblock %} -
-
+ diff --git a/digits/templates/login.html b/digits/templates/login.html new file mode 100644 index 000000000..ee81df044 --- /dev/null +++ b/digits/templates/login.html @@ -0,0 +1,29 @@ +{# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. #} + +{% from "helper.html" import print_flashes %} + +{% extends "layout.html" %} + +{% block content %} +
+

Login

+
+ +
+
+ {{ print_flashes() }} + +
+
+ + + + +
+ +
+
+
+ +{% endblock %} + diff --git a/digits/utils/__init__.py b/digits/utils/__init__.py index 1d479119c..c2d7300b6 100644 --- a/digits/utils/__init__.py +++ b/digits/utils/__init__.py @@ -154,5 +154,5 @@ def parse_version(*args): ### Import the other utility functions -from . import constants, image, time_filters, errors, forms +from . import constants, image, time_filters, errors, forms, routing, auth diff --git a/digits/utils/auth.py b/digits/utils/auth.py new file mode 100644 index 000000000..f138f9f8e --- /dev/null +++ b/digits/utils/auth.py @@ -0,0 +1,73 @@ +# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + +import flask +import functools +import re +import werkzeug.exceptions + +from .routing import get_request_arg, request_wants_json + +def get_username(): + return get_request_arg('username') or \ + flask.request.cookies.get('username', None) + +def validate_username(username): + """ + Raises a ValueError if the username is invalid + """ + if not username: + raise ValueError('username is required') + if not re.match('[a-z]', username): + raise ValueError('Must start with a lowercase letter') + if not re.match('[a-z0-9\.\-_]+$', username): + raise ValueError('Only lowercase letters, numbers, periods, dashes and underscores allowed') + +def requires_login(f=None, redirect=True): + """ + Decorator for views that require the user to be logged in + + Keyword arguments: + f -- the function to decorate + redirect -- if True, this function may return a redirect + """ + if f is None: + # optional arguments are handled strangely + return functools.partial(requires_login, redirect=redirect) + + @functools.wraps(f) + def decorated(*args, **kwargs): + username = get_username() + if not username: + # Handle missing username + if request_wants_json() or not redirect: + raise werkzeug.exceptions.Unauthorized() + else: + return flask.redirect(flask.url_for('login', next=flask.request.path)) + try: + # Validate username + validate_username(username) + except ValueError as e: + raise werkzeug.exceptions.BadRequest('Invalid username - %s' % e.message) + return f(*args, **kwargs) + return decorated + +def has_permission(job, action, username=None): + """ + Returns True if username can perform action on job + + Arguments: + job -- the Job in question + action -- the action in question + + Keyword arguments: + username -- the user in question (defaults to current user) + """ + if username is None: + username = get_username() + + if not username: + return False + if not job.username: + return True + return username == job.username + diff --git a/digits/views.py b/digits/views.py index 2f8291c4a..4e5d1c119 100644 --- a/digits/views.py +++ b/digits/views.py @@ -11,12 +11,11 @@ from flask.ext.socketio import join_room, leave_room import digits -from . import dataset, model +from digits import dataset, model, utils from config import config_value from webapp import app, socketio, scheduler import dataset.views import model.views -from digits.utils import errors from digits.utils.routing import request_wants_json from digits.log import logger @@ -99,6 +98,48 @@ def get_job_list(cls, running): ) +### Authentication/login + +@app.route('/login', methods=['GET','POST']) +def login(): + """ + Ask for a username (no password required) + Sets a cookie + """ + # Get the URL to redirect to after logging in + next_url = utils.routing.get_request_arg('next') or \ + flask.request.referrer or flask.url_for('home') + + if flask.request.method == 'GET': + return flask.render_template('login.html', next=next_url) + + # Validate username + username = utils.routing.get_request_arg('username').strip() + try: + utils.auth.validate_username(username) + except ValueError as e: + # Invalid username + flask.flash(e.message, 'danger') + return flask.render_template('login.html', next=next_url) + + # Valid username + response = flask.make_response(flask.redirect(next_url)) + response.set_cookie('username', username) + return response + +@app.route('/logout') +def logout(): + """ + Unset the username cookie + """ + next_url = utils.routing.get_request_arg('next') or \ + flask.request.referrer or flask.url_for('home') + + response = flask.make_response(flask.redirect(next_url)) + response.set_cookie('username', '', expires=0) + return response + + ### Jobs routes @app.route('/jobs/', methods=['GET']) @@ -178,7 +219,7 @@ def delete_job(job_id): return 'Job deleted.' else: raise werkzeug.exceptions.Forbidden('Job not deleted') - except errors.DeleteError as e: + except utils.errors.DeleteError as e: raise werkzeug.exceptions.Forbidden(str(e)) @app.route('/datasets//abort', methods=['POST']) diff --git a/digits/webapp.py b/digits/webapp.py index fb55b3e45..ba19b11b9 100644 --- a/digits/webapp.py +++ b/digits/webapp.py @@ -32,6 +32,18 @@ import digits.views +def username_decorator(f): + from functools import wraps + @wraps(f) + def decorated(*args, **kwargs): + this_username = flask.request.cookies.get('username', None) + app.jinja_env.globals['username'] = this_username + return f(*args, **kwargs) + return decorated + +for endpoint, function in app.view_functions.iteritems(): + app.view_functions[endpoint] = username_decorator(function) + ### Setup the environment scheduler.load_past_jobs() From ec9888e5079bfef20c4b11e3d54a6418961fa317 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Thu, 10 Dec 2015 13:47:00 -0800 Subject: [PATCH 3/3] Protect routes and buttons according to login info Update testsuite to match --- digits/dataset/images/classification/views.py | 3 +++ digits/dataset/images/generic/views.py | 8 ++++++-- digits/job.py | 6 +++++- digits/model/images/classification/views.py | 3 +++ digits/model/images/generic/views.py | 3 +++ digits/templates/job.html | 12 ++++++++---- digits/templates/job_item.html | 4 +++- digits/test_scheduler.py | 2 +- digits/test_status.py | 6 +++--- digits/test_views.py | 7 +++++++ digits/views.py | 12 +++++++++++- digits/webapp.py | 1 + 12 files changed, 54 insertions(+), 13 deletions(-) diff --git a/digits/dataset/images/classification/views.py b/digits/dataset/images/classification/views.py index b89ef1dda..4e836b243 100644 --- a/digits/dataset/images/classification/views.py +++ b/digits/dataset/images/classification/views.py @@ -254,6 +254,7 @@ def from_files(job, form): @app.route(NAMESPACE + '/new', methods=['GET']) +@utils.auth.requires_login def image_classification_dataset_new(): """ Returns a form for a new ImageClassificationDatasetJob @@ -267,6 +268,7 @@ def image_classification_dataset_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) +@utils.auth.requires_login(redirect=False) def image_classification_dataset_create(): """ Creates a new ImageClassificationDatasetJob @@ -287,6 +289,7 @@ def image_classification_dataset_create(): job = None try: job = ImageClassificationDatasetJob( + username = utils.auth.get_username(), name = form.dataset_name.data, image_dims = ( int(form.resize_height.data), diff --git a/digits/dataset/images/generic/views.py b/digits/dataset/images/generic/views.py index c78c1696a..2a6ec42e8 100644 --- a/digits/dataset/images/generic/views.py +++ b/digits/dataset/images/generic/views.py @@ -2,6 +2,7 @@ import flask +from digits import utils from digits.utils.forms import fill_form_if_cloned, save_form_to_job from digits.utils.routing import request_wants_json, job_from_request from digits.webapp import app, scheduler @@ -12,6 +13,7 @@ NAMESPACE = '/datasets/images/generic' @app.route(NAMESPACE + '/new', methods=['GET']) +@utils.auth.requires_login def generic_image_dataset_new(): """ Returns a form for a new GenericImageDatasetJob @@ -25,6 +27,7 @@ def generic_image_dataset_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) +@utils.auth.requires_login(redirect=False) def generic_image_dataset_create(): """ Creates a new GenericImageDatasetJob @@ -45,8 +48,9 @@ def generic_image_dataset_create(): job = None try: job = GenericImageDatasetJob( - name = form.dataset_name.data, - mean_file = form.prebuilt_mean_file.data.strip(), + username = utils.auth.get_username(), + name = form.dataset_name.data, + mean_file = form.prebuilt_mean_file.data.strip(), ) if form.method.data == 'prebuilt': diff --git a/digits/job.py b/digits/job.py index 91bcc1539..a15724afb 100644 --- a/digits/job.py +++ b/digits/job.py @@ -42,10 +42,11 @@ def load(cls, job_id): task.detect_snapshots() return job - def __init__(self, name): + def __init__(self, name, username): """ Arguments: name -- name of this job + username -- creator of this job """ super(Job, self).__init__() @@ -53,6 +54,7 @@ def __init__(self, name): self._id = '%s-%s' % (time.strftime('%Y%m%d-%H%M%S'), os.urandom(2).encode('hex')) self._dir = os.path.join(config_value('jobs_dir'), self._id) self._name = name + self.username = username self.pickver_job = PICKLE_VERSION self.tasks = [] self.exception = None @@ -76,6 +78,8 @@ def __setstate__(self, state): """ Used when loading a pickle file """ + if 'username' not in state: + state['username'] = None self.__dict__ = state def json_dict(self, detailed=False): diff --git a/digits/model/images/classification/views.py b/digits/model/images/classification/views.py index f16cf78f0..c50d2bfe0 100644 --- a/digits/model/images/classification/views.py +++ b/digits/model/images/classification/views.py @@ -24,6 +24,7 @@ NAMESPACE = '/models/images/classification' @app.route(NAMESPACE + '/new', methods=['GET']) +@utils.auth.requires_login def image_classification_model_new(): """ Return a form for a new ImageClassificationModelJob @@ -49,6 +50,7 @@ def image_classification_model_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) +@utils.auth.requires_login(redirect=False) def image_classification_model_create(): """ Create a new ImageClassificationModelJob @@ -86,6 +88,7 @@ def image_classification_model_create(): job = None try: job = ImageClassificationModelJob( + username = utils.auth.get_username(), name = form.model_name.data, dataset_id = datasetJob.id(), ) diff --git a/digits/model/images/generic/views.py b/digits/model/images/generic/views.py index 341ee1ec5..f8b1f0ebe 100644 --- a/digits/model/images/generic/views.py +++ b/digits/model/images/generic/views.py @@ -22,6 +22,7 @@ NAMESPACE = '/models/images/generic' @app.route(NAMESPACE + '/new', methods=['GET']) +@utils.auth.requires_login def generic_image_model_new(): """ Return a form for a new GenericImageModelJob @@ -46,6 +47,7 @@ def generic_image_model_new(): @app.route(NAMESPACE + '.json', methods=['POST']) @app.route(NAMESPACE, methods=['POST']) +@utils.auth.requires_login(redirect=False) def generic_image_model_create(): """ Create a new GenericImageModelJob @@ -82,6 +84,7 @@ def generic_image_model_create(): job = None try: job = GenericImageModelJob( + username = utils.auth.get_username(), name = form.model_name.data, dataset_id = datasetJob.id(), ) diff --git a/digits/templates/job.html b/digits/templates/job.html index cc182b256..e8ff0aa70 100644 --- a/digits/templates/job.html +++ b/digits/templates/job.html @@ -91,7 +91,9 @@

{{ job.name() }}

+ {% if job | has_permission('edit') %} + {% endif %}