From 1987735ed7a209f198a259f7dfca4a2868af5c2d Mon Sep 17 00:00:00 2001 From: jay-johnson Date: Sun, 1 Apr 2018 23:47:59 -0700 Subject: [PATCH] initial docs with automodule --- README.rst | 53 ++++++++++++++++++++++++++++ antinex_core/antinex_worker.py | 2 +- docs/source/api_reference.rst | 29 ++++++++++++++++ docs/source/index.rst | 63 ++++++++++++++++++++++++++++++++++ docs/source/scripts.rst | 36 +++++++++++++++++++ setup.py | 3 +- 6 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 docs/source/api_reference.rst create mode 100644 docs/source/scripts.rst diff --git a/README.rst b/README.rst index c5e46c2..b171c00 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,59 @@ This repository is a standalone training and prediction worker pool that is deco https://github.com/jay-johnson/train-ai-with-django-swagger-jwt +AntiNex Stack Status +-------------------- + +AntiNex Core Worker is part of the AntiNex stack: + +.. list-table:: + :header-rows: 1 + + * - Component + - Build + - Docs Link + - Docs Build + * - `REST API `_ + - .. image:: https://travis-ci.org/jay-johnson/train-ai-with-django-swagger-jwt.svg?branch=master + :alt: Travis Tests + :target: https://travis-ci.org/jay-johnson/train-ai-with-django-swagger-jwt.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex/badge/?version=latest + :alt: Read the Docs REST API Tests + :target: https://readthedocs.org/projects/antinex/badge/?version=latest + * - `Core Worker `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-core.svg?branch=master + :alt: Travis AntiNex Core Tests + :target: https://travis-ci.org/jay-johnson/antinex-core.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-core-worker/badge/?version=latest + :alt: Read the Docs AntiNex Core Tests + :target: http://antinex-core-worker.readthedocs.io/en/latest/?badge=latest + * - `Network Pipeline `_ + - .. image:: https://travis-ci.org/jay-johnson/network-pipeline.svg?branch=master + :alt: Travis AntiNex Network Pipeline Tests + :target: https://travis-ci.org/jay-johnson/network-pipeline.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-network-pipeline/badge/?version=latest + :alt: Read the Docs AntiNex Network Pipeline Tests + :target: https://readthedocs.org/projects/antinex-network-pipeline/badge/?version=latest + * - `AI Utils `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-utils.svg?branch=master + :alt: Travis AntiNex AI Utils Tests + :target: https://travis-ci.org/jay-johnson/antinex-utils.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-ai-utilities/badge/?version=latest + :alt: Read the Docs AntiNex AI Utils Tests + :target: http://antinex-ai-utilities.readthedocs.io/en/latest/?badge=latest + * - `Client `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-client.svg?branch=master + :alt: Travis AntiNex Client Tests + :target: https://travis-ci.org/jay-johnson/antinex-client.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-client/badge/?version=latest + :alt: Read the Docs AntiNex Client Tests + :target: https://readthedocs.org/projects/antinex-client/badge/?version=latest + Install ------- diff --git a/antinex_core/antinex_worker.py b/antinex_core/antinex_worker.py index 625a4a0..a979bd7 100644 --- a/antinex_core/antinex_worker.py +++ b/antinex_core/antinex_worker.py @@ -223,7 +223,7 @@ def shutdown( "100")) max_models = int(ev( "MAX_MODELS", - "10")) + "100")) log.info("Creating antinex core") core = AntiNexCore( diff --git a/docs/source/api_reference.rst b/docs/source/api_reference.rst new file mode 100644 index 0000000..a4cd49b --- /dev/null +++ b/docs/source/api_reference.rst @@ -0,0 +1,29 @@ +=================================== +AntiNex Core Worker - API Reference +=================================== + +Celery Worker +============= + +Here is the Celery Worker's source code. + +.. automodule:: antinex_core.antinex_worker + :members: setup_celery_logging,AntiNexCore,app,core,broker_url,train_queue_name,predict_queue_name,max_msgs,max_models + +Process Consumed Messages From the Queues +========================================= + +The processor class processes any messages the worker consumes from the queue. + +.. automodule:: antinex_core.antinex_processor + :members: AntiNexProcessor + +Send Results to the Broker +========================== + +This method is responsible for publishing what the core's results were from the processed job. + +.. note:: The results must be sent back as a JSON dictionary for the REST API's Celery Workers to handle. + +.. automodule:: antinex_core.send_results_to_broker + :members: send_results_to_broker diff --git a/docs/source/index.rst b/docs/source/index.rst index fa983a1..61c7d26 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,9 +6,72 @@ AntiNex Core Worker docs ======================== +AntiNex Stack Status +-------------------- + +AntiNex Core Worker is part of the AntiNex stack: + +.. list-table:: + :header-rows: 1 + + * - Component + - Build + - Docs Link + - Docs Build + * - `REST API `_ + - .. image:: https://travis-ci.org/jay-johnson/train-ai-with-django-swagger-jwt.svg?branch=master + :alt: Travis Tests + :target: https://travis-ci.org/jay-johnson/train-ai-with-django-swagger-jwt.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex/badge/?version=latest + :alt: Read the Docs REST API Tests + :target: https://readthedocs.org/projects/antinex/badge/?version=latest + * - `Core Worker `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-core.svg?branch=master + :alt: Travis AntiNex Core Tests + :target: https://travis-ci.org/jay-johnson/antinex-core.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-core-worker/badge/?version=latest + :alt: Read the Docs AntiNex Core Tests + :target: http://antinex-core-worker.readthedocs.io/en/latest/?badge=latest + * - `Network Pipeline `_ + - .. image:: https://travis-ci.org/jay-johnson/network-pipeline.svg?branch=master + :alt: Travis AntiNex Network Pipeline Tests + :target: https://travis-ci.org/jay-johnson/network-pipeline.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-network-pipeline/badge/?version=latest + :alt: Read the Docs AntiNex Network Pipeline Tests + :target: https://readthedocs.org/projects/antinex-network-pipeline/badge/?version=latest + * - `AI Utils `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-utils.svg?branch=master + :alt: Travis AntiNex AI Utils Tests + :target: https://travis-ci.org/jay-johnson/antinex-utils.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-ai-utilities/badge/?version=latest + :alt: Read the Docs AntiNex AI Utils Tests + :target: http://antinex-ai-utilities.readthedocs.io/en/latest/?badge=latest + * - `Client `_ + - .. image:: https://travis-ci.org/jay-johnson/antinex-client.svg?branch=master + :alt: Travis AntiNex Client Tests + :target: https://travis-ci.org/jay-johnson/antinex-client.svg + - `Docs `_ + - .. image:: https://readthedocs.org/projects/antinex-client/badge/?version=latest + :alt: Read the Docs AntiNex Client Tests + :target: https://readthedocs.org/projects/antinex-client/badge/?version=latest + +Table of Contents +================= + +These are the docs for the AntiNex Core Worker repository. + +Source Code +----------- + .. toctree:: :maxdepth: 2 + api_reference + scripts Indices and tables ================== diff --git a/docs/source/scripts.rst b/docs/source/scripts.rst new file mode 100644 index 0000000..d98c737 --- /dev/null +++ b/docs/source/scripts.rst @@ -0,0 +1,36 @@ +======= +Scripts +======= + +Standalone Processing Examples +============================== + +Using Scaler Train and Test Helper +---------------------------------- + +Train a DNN using the Scaler-Normalized AntiNex Django Dataset. This builds the train and test datasets using the ``build_scaler_train_and_test_datasets`` method from the internal modules. + +.. automodule:: antinex_core.scripts.antinex-scaler-django + :members: dataset,model_backup_file,model_weights_file,model_json_file,model_image_file,footnote_text,image_title,show_predictions,features_to_process,min_scaler_range,max_scaler_range,test_size,batch_size,epochs,num_splits,loss,optimizer,metrics,histories,label_rules,build_model,model,scaler_res,train_scaler_df,sample_rows,kfold,results,scores,cm,sample_predictions,merged_predictions_df,predict_rows_df,fig,ax + +Using Manual Scaler Objects +--------------------------- + +Train a DNN using the Scaler-Normalized AntiNex Django Dataset. This builds the train and test datasets manually to verify the process before editing the ``build_scaler_train_and_test_datasets`` method. + +.. automodule:: antinex_core.scripts.standalone-scaler-django + :members: dataset,model_backup_file,model_weights_file,model_json_file,model_image_file,datanode,footnote_text,image_title,show_predictions,features_to_process,min_scaler_range,max_scaler_range,test_size,batch_size,epochs,num_splits,loss,optimizer,metrics,histories,label_rules,build_model,model,scaler_res,train_scaler_df,sample_rows,kfold,results,scores,cm,sample_predictions,merged_predictions_df,predict_rows_df,test_df,train_df,test_scaler,train_scaler,train_only_floats,test_only_floats,fig,ax + +Convert Bottom Rows from a CSV File into JSON +============================================= + +When testing live DNN predictions you can use this utility script to print a few JSON-ready dictionaries out to ``stdout``. + +Usage: + +:: + + convert-bottom-rows-to-json.py -f -b + +.. automodule:: antinex_core.scripts.convert-bottom-rows-to-json + :members: parser,dataset,bottom_row_idx,output_predict_rows diff --git a/setup.py b/setup.py index 7ed9599..df3c14f 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def run_tests(self): setup( name="antinex-core", cmdclass={"test": PyTest}, - version="1.0.25", + version="1.0.26", description=("AntiNex publisher-subscriber core for processing " "training and prediction requests for deep neural " "networks to detect network exploits using Keras " @@ -99,6 +99,7 @@ def run_tests(self): url="https://github.com/jay-johnson/antinex-core", packages=[ "antinex_core", + "antinex_core.scripts", "antinex_core.log" ], package_data={},