data update

docs/_static/files/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2'
 
-    services:
+services:
     client:
         image: maestroserver/client-maestro
         ports:
@@ -13,70 +13,70 @@ version: '2'
         ports:
         - "8888:8888"
         environment:
-        - "MAESTRO_PORT=8888"
         - "MAESTRO_MONGO_URI=mongodb/maestro-client"
-        - "MAESTRO_DISCOVERY_TIMEOUT=10000"
         - "MAESTRO_DISCOVERY_URL=http://discovery:5000"
+        - "MAESTRO_REPORT_URL=http://reports:5000"
 
     discovery:
         image: maestroserver/discovery-maestro
         ports:
         - "5000:5000"
         environment:
         - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-        - "MAESTRO_PORT=5000"
-        - "MAESTRO_MONGO_URI=mongodb"
-        - "MAESTRO_MONGO_DATABASE=maestro-client"
+        - "MAESTRO_DATA_URL=http://data:5000"
 
-    celery:
+    discovery-celery:
         image: maestroserver/discovery-maestro-celery
         environment:
-        - "MAESTRO_DISCOVERY_URL=http://discovery"
-        - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-        - "MAESTRO_PORT=5000"
+        - "MAESTRO_DATA_URL=http://data:5000"
+        - "CELERY_BROKER_URL=amqp://rabbitmq:5672" 
 
-  reports:
-    image: maestroserver/reports-maestro
-    environment:
-     - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-     - "MAESTRO_URL=http://localhost:5005"
-     - "MAESTRO_MONGO_URI=mongodb"
-     - "MAESTRO_MONGO_DATABASE=maestro-reports"
+    reports:
+        image: maestroserver/reports-maestro
+        environment:
+        - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
+        - "MAESTRO_MONGO_URI=mongodb"
+        - "MAESTRO_MONGO_DATABASE=maestro-reports"
 
-  reports_worker:
-    image: maestroserver/reports-maestro-celery
-    environment:
-     - "MAESTRO_URL=http://reports:5005"
-     - "MAESTRO_DISCOVERY_URL=http://discovery:5000"
-     - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
+    reports_worker:
+        image: maestroserver/reports-maestro-celery
+        environment:
+        - "MAESTRO_DATA_URI=http://data:5000"
+        - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
 
     scheduler:
         image: maestroserver/scheduler-maestro
         environment:
-        - "MAESTRO_DISCOVERY_URL=http://discovery"
-        - "MAESTRO_DISCOVERY_PORT=5000"
+        - "MAESTRO_DATA_URI=http://data:5000"
         - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
         - "MAESTRO_MONGO_URI=mongodb"
         - "MAESTRO_MONGO_DATABASE=maestro-client"
 
+    scheduler_worker:
+        image: maestroserver/scheduler-maestro-celery
+        environment:
+        - "MAESTRO_DATA_URI=http://data:5000"
+        - "CELERY_BROKER_URL=amqp://rabbitmq:5672"   
+
+    data:
+        image: maestroserver/data-maestro
+        environment:
+            - "MAESTRO_MONGO_URI=mongodb"
+            - "MAESTRO_MONGO_DATABASE=maestro-client"
+
     rabbitmq:
         hostname: "discovery-rabbit"
         image: rabbitmq:3-management
         ports:
         - "15672:15672"
         - "5672:5672"
-
-    redis:
-        image: redis
-        ports:
-        - "6379:6379"
-
+
     mongodb:
         image: mongo
         volumes:
         - mongodata:/data/db
         ports:
         - "27017:27017"
 
-    volumes:
-    mongodata: {}
+volumes:
+mongodata: {}

docs/developer/api/data.rst

@@ -0,0 +1,8 @@
+Data Layer API
+--------------
+
+Data Layer API. `See docs data layer api <https://maestro-server.github.io/data-app/>`_.
+
+.. raw:: html
+
+    <iframe src="https://maestro-server.github.io/data-app/" height="645px" width="100%"></iframe>

docs/developer/api/index.rst

@@ -7,4 +7,5 @@ All commands, jobs, tasks or action is made by rest api, you can use se same api
    :maxdepth: 1
 
    server
-   discovery
+   discovery
+   data

docs/developer/architecture/data_app.rst

@@ -5,5 +5,28 @@ Data APP
 Data app, database gateway micro service
 - Request and response database operations
 
-.. Danger::
-	This service is under development
+Simple Rest API using `Flask <http://flask.pocoo.org>`_ (python) + pymongo.
+
+**Setup dev env**
+
+.. code-block:: bash
+
+    pip install
+
+	FLASK_APP=run.py FLASK_DEBUG=1 flask run --port=5010
+
+	or
+
+	npm run server
+
+---------------
+
+Mongo service
+
+.. code-block:: bash
+
+    cd devtool/
+
+    docker-compose up -d
+
+Will be setup mongodb

docs/developer/architecture/discovery_app.rst

@@ -47,10 +47,12 @@ If you use windows, celery havent support for windows, the last version is 3.1.2
 
 **Flower - Debbug Celery**
 
-We installed a flower, it's a control panel to centralize results throughout rabbitMQ, very useful to troubleshooting producer and consumers.
+You can install a flower, it's a control panel to centralize results throughout rabbitMQ, very useful to troubleshooting producer and consumers.
 
 .. code-block:: bash
 
+    pip install flower
+
     flower -A app.celery
 
     npm run flower

docs/developer/architecture/scheduler_app.rst

@@ -2,6 +2,36 @@
 Scheduler App
 -------------
 
+Scheduler use celery beat to control scheduler jobs, customized scheduler class based at `Zmap CeleryBeat Mongo <https://github.com/zmap/celerybeat-mongo>`_.
+
 .. image:: ../../_static/screen/scheduler.png
 
-Discovery using `Flask <http://flask.pocoo.org>`_,  and python >3.5, used Celery Beat feature to call tasks.
+**Installation with python 3**
+
+    - Python >3.4
+    - RabbitMQ
+    - MongoDB
+
+Download de repository
+
+.. code-block:: bash
+
+    git clone https://github.com/maestro-server/scheduler-app.git
+
+**Important topics**
+
+- Celery Beat consult schedulers collection in mongodb every 5 seconds and updated time to call the tasks.
+
+- Have 2 tasks called by beat
+
+	- **webhook:** Call HTTP request accordly arguments. 
+
+	- **connection:** Consulting connection data, after call webhook.
+
+- Have support tasks called by outhers tasks.
+
+	- **chain and chain_exec:** Called by webhook, this create another job after the first finish.
+
+	- **depleted_job:** If any job recevied something wrong, this taks is called e depleted that job.
+
+    - **notify_event:** Send notification event. 

docs/developer/qa.rst

@@ -114,3 +114,24 @@ CodeClimate       .. image:: https://api.codeclimate.com/v1/badges/3a073f54d89d9
                         :alt: Test Coverage
 ================  ================================================================================================================================================================================== 
 
+------------
+
+Data Layer API
+-----------------
+
+================  ================================================================================================================================================================================== 
+
+Codacy            .. image:: https://api.codacy.com/project/badge/Grade/d8b11776962a4867a491c7a039c250ec
+                        :target: https://www.codacy.com/app/maestro/data-app?utm_source=github.com&utm_medium=referral&utm_content=maestro-server/data-app&utm_campaign=Badge_Grade
+
+Travis            .. image:: https://travis-ci.org/maestro-server/data-app.svg?branch=master
+                        :target: https://travis-ci.org/maestro-server/data-app
+
+CodeClimate       .. image:: https://api.codeclimate.com/v1/badges/0d6930d08ca210a9e46e/maintainability
+                        :target: https://codeclimate.com/github/maestro-server/data-app/maintainability
+                        :alt: Maintainability
+                  .. image:: https://api.codeclimate.com/v1/badges/0d6930d08ca210a9e46e/test_coverage
+                        :target: https://codeclimate.com/github/maestro-server/data-app/test_coverage
+                        :alt: Test Coverage
+================  ================================================================================================================================================================================== 
+

docs/developer/tests.rst

@@ -61,3 +61,14 @@ Testing use pytest
     npm run test
 
     python -m unittest discover
+
+Data Layer APP
+--------------
+
+Testing use pytest
+
+.. code-block:: bash
+
+    npm run test
+
+    python -m unittest discover

docs/installing/data_app.rst

@@ -5,5 +5,75 @@ Data APP
 Data app, database gateway micro service
 - Request and response database operations
 
-.. Danger::
-	This service is under development
+----------
+
+**Installation by docker**
+
+.. code-block:: yaml
+
+	version: '2'
+
+	services:
+		data:
+			image: maestroserver/data-maestro
+			environment:
+				- "MAESTRO_MONGO_URI=mongodb"
+				- "MAESTRO_MONGO_DATABASE=maestro-client"
+
+----------
+
+**Installation with python 3**
+
+    - Python >3.4
+    - MongoDB
+
+Download de repository
+
+.. code-block:: bash
+
+    git clone https://github.com/maestro-server/data-app.git
+
+----------
+
+**Install  run api**
+
+.. code-block:: bash
+
+    python -m flask run.py --port 5010 
+
+    or
+
+    FLASK_APP=run.py FLASK_DEBUG=1 flask run --port 5010 
+
+    or 
+
+    npm run server
+
+----------
+
+.. Warning::
+
+    For production environment, use something like gunicorn.
+
+.. code-block:: python
+
+	# gunicorn_config.py
+
+	import os
+
+	bind = "0.0.0.0:" + str(os.environ.get("MAESTRO_PORT", 5010))
+	workers = os.environ.get("MAESTRO_GWORKERS", 2)
+
+----------
+
+**Env variables**
+
+======================= ============================ ===========================================
+Env Variables                   Example                    Description         
+======================= ============================ ===========================================
+MAESTRO_PORT			5000						 Port used 
+MAESTRO_MONGO_URI       localhost                    Mongo Url conn
+MAESTRO_MONGO_DATABASE  maestro-reports              Db name, its differente of servers-app     
+MAESTRO_GWORKERS   		2       					 Gunicorn multi process  
+MAESTRO_INSERT_QTD      200                          Throughput insert in reports collection
+======================= ============================ ===========================================

docs/installing/reports_app.rst

@@ -21,14 +21,13 @@ Reports app, generate reports
             image: maestroserver/reports-maestro
             environment:
             - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-            - "MAESTRO_URL=http://localhost:5005"
             - "MAESTRO_MONGO_URI=mongodb"
             - "MAESTRO_MONGO_DATABASE=maestro-reports"
 
         reports_worker:
             image: maestroserver/reports-maestro-celery
             environment:
-            - "MAESTRO_URL=http://reports:5005"
+            - "MAESTRO_DATA_URI=http://data:5000"
             - "MAESTRO_DISCOVERY_URL=http://discovery:5000"
             - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
 
@@ -99,7 +98,7 @@ Env Variables                   Example                    Description
 MAESTRO_MONGO_URI       localhost                    Mongo Url conn
 MAESTRO_MONGO_DATABASE  maestro-reports              Db name, its differente of servers-app     
 MAESTRO_DISCOVERY_URL   http://localhost:5000        Discovery API URL
-MAESTRO_URL             http://localhost:5005        Report api
+MAESTRO_REPORT_URI      http://localhost:5005        Report api
 MAESTRO_INSERT_QTD      200                          Throughput insert in reports collection
 CELERY_BROKER_URL       amqp://rabbitmq:5672         RabbitMQ connection
 ======================= ============================ ===========================================

docs/installing/scheduler_app.rst

@@ -24,16 +24,14 @@ Scheduler App service to manage and execute jobs
             environment:
             - "MAESTRO_DATA_URI=http://data:5000"
             - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-            - "MAESTRO_MONGO_URI=localhost"
+            - "MAESTRO_MONGO_URI=mongodb"
             - "MAESTRO_MONGO_DATABASE=maestro-client"
 
-        scheduler:
+        scheduler_worker:
             image: maestroserver/scheduler-maestro-celery
             environment:
             - "MAESTRO_DATA_URI=http://data:5000"
             - "CELERY_BROKER_URL=amqp://rabbitmq:5672"
-            - "MAESTRO_MONGO_URI=localhost"
-            - "MAESTRO_MONGO_DATABASE=maestro-client"
 
  Run docker compose
 

docs/installing/server_app.rst

@@ -26,6 +26,7 @@ Server App main application, some responsibility is
         environment:
         - "MAESTRO_MONGO_URI=mongodb/maestro-client"
         - "MAESTRO_DISCOVERY_URL=http://discovery:5000"
+        - "MAESTRO_REPORT_URL=http://reports:5000"
 
 Run docker compose
 
@@ -37,7 +38,7 @@ Or docker run
 
 .. code-block:: bash
 
-    docker run -p 8888:8888  -e "MAESTRO_MONGO_URI=mongodb/maestro-client" -e "MAESTRO_DISCOVERY_URL=http://discovery:5000" maestroserver/server-maestro 
+    docker run -p 8888:8888  -e "MAESTRO_MONGO_URI=mongodb/maestro-client" -e "MAESTRO_DISCOVERY_URL=http://discovery:5000" -e "MAESTRO_REPORT_URL=http://reports:5000" maestroserver/server-maestro 
 
 ----------