fix(daemon): fix some helper text for redocs (#1642)

* fix(daemon): fix some helper text for redocs

.github/workflows/cd.yml

@@ -59,6 +59,7 @@ jobs:
       - name: push-to-api-repo
         run: |
           cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html
+          cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html
           cd schema
           git config --local user.email "dev-bot@jina.ai"
           git config --local user.name "Jina Dev Bot"

.github/workflows/tag.yml

@@ -38,6 +38,8 @@ jobs:
         run: |
           cp jinad.html schema/jinad/index.html
           cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html
+          cp jinad.html schema/daemon/index.html
+          cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html
           cd schema
           git config --local user.email "dev-bot@jina.ai"
           git config --local user.name "Jina Dev Bot"

daemon/__init__.py

@@ -1,4 +1,5 @@
 import json
+import os
 import subprocess
 import threading
 from collections import namedtuple
@@ -10,7 +11,9 @@
 from jina.logging import JinaLogger
 from .parser import get_main_parser
 
-daemon_logger = JinaLogger(context='👻 JINAD')
+daemon_logger = JinaLogger(context='👻', log_config=os.getenv('JINAD_LOG_CONFIG',
+                                                              pkg_resources.resource_filename('jina', '/'.join(
+                                                                  ('resources', 'logging.daemon.yml')))))
 
 
 def _get_app():
@@ -34,21 +37,17 @@ def _get_app():
         description=fastapi_config.DESCRIPTION,
         version=fastapi_config.VERSION
     )
-    app.include_router(router=common_router,
-                       prefix=fastapi_config.PREFIX)
-    app.include_router(router=logs.router,
-                       prefix=fastapi_config.PREFIX)
+    app.include_router(router=common_router)
+    app.include_router(router=logs.router)
     if jinad_config.CONTEXT == 'all':
         for _current_router in _all_routers.values():
             app.include_router(router=_current_router.router,
-                               tags=_current_router.tags,
-                               prefix=fastapi_config.PREFIX)
+                               tags=_current_router.tags)
     else:
         _current_router = _all_routers[jinad_config.CONTEXT]
         app.openapi_tags = _current_router.openapi_tags
         app.include_router(router=_current_router.router,
-                           tags=_current_router.tags,
-                           prefix=fastapi_config.PREFIX)
+                           tags=_current_router.tags)
     return app
 
 

daemon/api/endpoints/__init__.py

@@ -9,13 +9,13 @@
 
 @common_router.on_event('startup')
 async def startup():
-    daemon_logger.success('Welcome to Jina daemon - the manager of distributed Jina')
-    daemon_logger.success(f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}')
+    daemon_logger.success(f'Welcome to Jina daemon - the manager of distributed Jina\n'
+                          f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}')
 
 
 @common_router.get(
     path='/alive',
-    summary='Get status of jinad',
+    summary='Check if daemon is alive',
     status_code=status.HTTP_200_OK
 )
 async def _status():

daemon/api/endpoints/flow.py

@@ -10,15 +10,14 @@
 from jina.parsers import set_client_cli_parser
 from ...excepts import FlowYamlParseException, FlowCreationException, FlowStartException
 from ...models import SinglePodModel
-from ...models.custom import build_pydantic_model
 from ...store import flow_store
 
 router = APIRouter()
 
 
 @router.put(
     path='/flow/pods',
-    summary='Build & start a Flow using Pods',
+    summary='Start a Flow from list of Pods',
 )
 async def _create_from_pods(
         pods: Union[List[Dict]] = Body(..., example=json.loads(SinglePodModel().json()))
@@ -59,7 +58,7 @@ async def _create_from_pods(
 
 @router.put(
     path='/flow/yaml',
-    summary='Build & start a Flow using YAML',
+    summary='Start a Flow from a YAML config',
 )
 async def _create_from_yaml(
         yamlspec: UploadFile = File(...),
@@ -69,26 +68,24 @@ async def _create_from_yaml(
     """
     Build a flow using [Flow YAML](https://docs.jina.ai/chapters/yaml/yaml.html#flow-yaml-sytanx)
 
-    > Upload Flow yamlspec (`yamlspec`)
-
-    > Yamls that Pods use (`uses_files`) (Optional)
-
-    > Python modules (`pymodules_files`) that the Pods use (Optional)
+    - Upload Flow yamlspec (`yamlspec`)
+    - Yamls that Pods use (`uses_files`) (Optional)
+    - Python modules (`pymodules_files`) that the Pods use (Optional)
 
     **yamlspec**:
 
         !Flow
+        version: 1.0
         with:
-            rest_api: true
-            compress_hwm: 1024
+            restful: true
         pods:
-            encode:
-                uses: helloworld.encoder.yml
-                parallel: 2
-            index:
-                uses: helloworld.indexer.yml
-                shards: 2
-                separated_workspace: true
+            - name: encode
+              uses: helloworld.encoder.yml
+              parallel: 2
+            - name: index
+              uses: helloworld.indexer.yml
+              shards: 2
+              separated_workspace: true
 
     **uses_files**: `helloworld.encoder.yml`
 
@@ -158,7 +155,7 @@ def __init__(self):
 
 @router.get(
     path='/flow/{flow_id}',
-    summary='Get Flow information',
+    summary='Get the status of a running Flow',
 )
 async def _fetch(
         flow_id: uuid.UUID,
@@ -193,7 +190,7 @@ async def _fetch(
 
 @router.get(
     path='/ping',
-    summary='Connect to Flow gateway',
+    summary='Check if the Flow is alive',
 )
 async def _ping(
         host: str,
@@ -208,6 +205,7 @@ async def _ping(
     _, args, _ = ArgNamespace.get_parsed_args(kwargs, set_client_cli_parser())
     client = Client(args)
     try:
+        # TODO: this introduces side-effect, need to be refactored. (2020.01.10)
         client.index(input_fn=['abc'])
         return {
             'status_code': status.HTTP_200_OK,
@@ -220,7 +218,7 @@ async def _ping(
 
 @router.delete(
     path='/flow',
-    summary='Close Flow Context',
+    summary='Terminate a running Flow',
 )
 async def _delete(
         flow_id: uuid.UUID

daemon/api/endpoints/logs.py

@@ -105,5 +105,7 @@ async def on_disconnect(self, websocket: WebSocket, close_code: int) -> None:
         daemon_logger.info(f'Client {self.client_details} got disconnected!')
 
 
+# TODO: adding websocket in this way do not generate any docs
+#  see: https://github.com/tiangolo/fastapi/issues/1983
 router.add_websocket_route(path='/logstream/{log_id}',
                            endpoint=LogStreamingEndpoint)

daemon/api/endpoints/pea.py

@@ -15,7 +15,7 @@
 
 @router.put(
     path='/pea/upload',
-    summary='Upload pod context yamls & pymodules',
+    summary='Upload YAML & py_modules required by a Pea',
 )
 async def _upload(
         uses_files: List[UploadFile] = File(()),
@@ -70,7 +70,7 @@ async def _create(
 
 @router.delete(
     path='/pea',
-    summary='Close Pea Context',
+    summary='Terminate a running Pea',
 )
 async def _delete(
         pea_id: uuid.UUID

daemon/api/endpoints/pod.py

@@ -15,7 +15,7 @@
 
 @router.put(
     path='/upload',
-    summary='Upload pod context yamls & pymodules',
+    summary='Upload YAML & py_modules required by a Pod',
 )
 async def _upload(
         uses_files: List[UploadFile] = File(()),
@@ -41,7 +41,7 @@ async def _upload(
 
 @router.put(
     path='/pod',
-    summary='Create a Pod via Flow or CLI',
+    summary='Create a Pod',
 )
 async def _create(
         pod_arguments: Union[SinglePodModel, ParallelPodModel]
@@ -71,7 +71,7 @@ async def _create(
 
 @router.delete(
     path='/pod',
-    summary='Close Pod Context',
+    summary='Terminate a running Pod',
 )
 async def _delete(
         pod_id: uuid.UUID

daemon/config.py

@@ -2,49 +2,46 @@
 
 from jina import __version__
 
-__prefix__ = 'v1'
-
 
 class BaseConfig(BaseSettings):
     class Config:
         env_prefix = 'JINAD_'
 
 
 class FastAPIConfig(BaseConfig):
-    NAME: str = 'Jina Remote Manager'
-    DESCRIPTION: str = 'REST API for managing Jina on Remote'
+    NAME: str = 'JinaD (Daemon)'
+    DESCRIPTION: str = 'The REST API of the daemon for managing distributed Jina'
     VERSION: str = __version__
-    PREFIX: str = '/' + __prefix__
 
 
 class OpenAPITags(BaseConfig):
     API_TAGS: list = [{
-        'name': 'Jina Remote Management',
-        'description': 'API to invoke remote Flows/Pods/Peas',
+        'name': 'Distributed Jina management',
+        'description': 'API to invoke distributed Flows/Pods/Peas',
         'externalDocs': {
-            'description': 'Jina Remote Context Manager',
+            'description': 'Jina Documentation',
             'url': 'https://docs.jina.ai/',
         },
     }]
     FLOW_API_TAGS: list = [{
-        'name': 'Remote Flow Manager',
-        'description': 'API to invoke local/remote Flows',
+        'name': 'Managing distributed Flow',
+        'description': 'API to invoke distributed Flows',
         'externalDocs': {
             'description': 'Jina Flow Context Manager',
             'url': 'https://docs.jina.ai/chapters/flow/index.html',
         },
     }]
     POD_API_TAGS: list = [{
-        'name': 'Remote Pod Manager',
-        'description': 'API to invoke remote Pods (__should be used by Flow APIs only__)',
+        'name': 'Managing distributed Pod',
+        'description': 'API to invoke distributed Pods (__should be used by Flow APIs only__)',
         'externalDocs': {
             'description': 'Jina 101',
             'url': 'https://docs.jina.ai/chapters/101/.sphinx.html',
         },
     }]
     PEA_API_TAGS: list = [{
-        'name': 'Remote Pea Manager',
-        'description': 'API to invoke remote Peas',
+        'name': 'Managing distributed Pea',
+        'description': 'API to invoke distributed Peas',
         'externalDocs': {
             'description': 'Jina 101',
             'url': 'https://docs.jina.ai/chapters/101/.sphinx.html',

daemon/store.py

@@ -6,12 +6,11 @@
 
 from fastapi import UploadFile
 
+from jina.enums import PodRoleType
 from jina.flow import Flow
-from jina.jaml import JAML
 from jina.helper import colored
-from jina.enums import PodRoleType
-from jina.logging import JinaLogger
 from jina.peapods import Pea, Pod
+from . import daemon_logger
 from .excepts import FlowYamlParseException, FlowCreationException, \
     FlowStartException, PodStartException, PeaStartException, FlowBadInputException
 from .helper import create_meta_files_from_upload, delete_meta_files_from_upload
@@ -24,7 +23,7 @@ class InMemoryStore:
     # https://github.com/jina-ai/jinad/issues/4
     credentials = 'foo:bar'
     _session_token = None
-    logger = JinaLogger(context='🏪 STORE')
+    logger = daemon_logger
 
     @contextmanager
     def _session(self):
@@ -79,8 +78,7 @@ def _create(self,
         if isinstance(config, str) or isinstance(config, SpooledTemporaryFile):
             yamlspec = config.read().decode() if isinstance(config, SpooledTemporaryFile) else config
             try:
-                JAML.register(Flow)
-                flow = JAML.load(yamlspec)
+                flow = Flow.load_config(yamlspec)
             except Exception as e:
                 self.logger.error(f'Got error while loading from yaml {e!r}')
                 raise FlowYamlParseException

jina/peapods/runtimes/jinad/api.py

@@ -74,7 +74,6 @@ class JinadAPI:
     def __init__(self,
                  host: str,
                  port: int,
-                 version: str = 'v1',
                  logger: 'JinaLogger' = None,
                  timeout: int = 5, **kwargs):
         """
@@ -92,9 +91,9 @@ def __init__(self,
         # TODO: for https, the jinad server would need a tls certificate.
         # no changes would be required in terms of how the api gets invoked,
         # as requests does ssl verfication. we'd need to add some exception handling logic though
-        url = f'{host}:{port}/{version}'
-        rest_url = f'http://{url}'
-        websocket_url = f'ws://{url}'
+        base_url = f'{host}:{port}'
+        rest_url = f'http://{base_url}'
+        websocket_url = f'ws://{base_url}'
         self.alive_url = f'{rest_url}/alive'
         self.upload_url = f'{rest_url}/upload'
         self.pea_url = f'{rest_url}/pea'

jina/resources/logging.daemon.yml

@@ -0,0 +1,30 @@
+handlers:  # enabled handlers, order does not matter
+  - StreamHandler
+  - SysLogHandler
+  - FluentHandler
+level: INFO  # set verbose level
+configs:
+  FileHandler:
+    format: '%(asctime)s:{name}|%(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
+    output: 'daemon-{uptime}.log'
+    formatter: JsonFormatter
+  StreamHandler:
+    format: '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
+    formatter: ColorFormatter
+  SysLogHandler:
+    ident: # this will be prepend to all messages
+    format: '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
+    host: # when not given then record it locally, /dev/log on linux /var/run/syslog on mac
+    port: # when not given then record it locally,  /dev/log on linux /var/run/syslog on mac
+    formatter: PlainFormatter
+  FluentHandler:
+    # this configuration describes where is the fluentD daemon running and waiting for logs to be emitted.
+    # FluentD then will have its own configuration to forward the messages according to its own syntax
+    # prefix will help fluentD filter data. This will be prepended for FluentD to easily filter incoming messages
+    tag: daemon
+    host: 0.0.0.0
+    port: 24224
+    format:
+      host: '%(hostname)s'
+      process: '%(process)s'
+      type: '%(levelname)s'