feat: remove file logs for CLI

biothings/hub/__init__.py

@@ -105,18 +105,18 @@ def _config_for_app(config_mod=None):
 #     _config_for_app(_config)
 
 
-from biothings.utils.common import get_class_from_classpath
-from biothings.utils.hub import (
+from biothings.utils.common import get_class_from_classpath  # noqa: E402
+from biothings.utils.hub import (  # noqa: E402
     AlreadyRunningException,
     CommandDefinition,
     CommandError,
     HubShell,
     get_hub_reloader,
     pending,
 )
-from biothings.utils.jsondiff import make as jsondiff
-from biothings.utils.loggers import ShellLogger, WSLogHandler, WSShellHandler, get_logger
-from biothings.utils.version import check_new_version, get_version
+from biothings.utils.jsondiff import make as jsondiff  # noqa: E402
+from biothings.utils.loggers import ShellLogger, WSLogHandler, WSShellHandler, get_logger  # noqa: E402
+from biothings.utils.version import check_new_version, get_version  # noqa: E402
 
 # adjust some loggers...
 if os.environ.get("HUB_VERBOSE", "0") != "1":

biothings/hub/databuild/builder.py

@@ -255,7 +255,7 @@ def get_pinfo(self):
 
     def setup_log(self):
         log_name = self.target_name or self.build_name
-        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", log_name)
+        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", log_name) if btconfig.LOG_FOLDER else None
         self.logger, _ = get_logger("build", log_folder=log_folder, force=True)
 
     def check_ready(self, force=False):

biothings/hub/databuild/differ.py

@@ -53,7 +53,7 @@ def setup_log(self, old=None, new=None):
         log_folder = self.log_folder
         name = f"diff_{self.__class__.diff_type}"
         if old and new:
-            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", new, "diff")
+            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", new, "diff") if btconfig.LOG_FOLDER else None
             name = f"diff_{old}_{new}"
         self.logger, self.logfile = get_logger(name, log_folder, force=True)
 

biothings/hub/databuild/syncer.py

@@ -56,7 +56,7 @@ def __init__(self, job_manager, log_folder):
     def setup_log(self, build_name=None):
         log_folder = None
         if build_name:
-            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name)
+            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name) if btconfig.LOG_FOLDER else None
         self.logger, self.logfile = get_logger("sync", log_folder=log_folder, force=True)
 
     def get_predicates(self):

biothings/hub/dataindex/indexer.py

@@ -318,7 +318,7 @@ def __init__(self, build_doc, indexer_env, index_name):
         self.pinfo = ProcessInfo(self, indexer_env.get("concurrency", 10))
 
     def setup_log(self):
-        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", self.build_name or "", "index")
+        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", self.build_name or "", "index") if btconfig.LOG_FOLDER else None
         log_name = f"index_{self.es_index_name}"
         self.logger, self.logfile = get_logger(log_name, log_folder=log_folder, force=True)
 

biothings/hub/dataindex/snapshooter.py

@@ -193,7 +193,7 @@ def _doc(self, index):
     def setup_log(self, index):
         build_doc = self._doc(index)
         log_name = build_doc["target_name"] or build_doc["_id"]
-        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", log_name, "snapshot")
+        log_folder = os.path.join(btconfig.LOG_FOLDER, "build", log_name, "snapshot") if btconfig.LOG_FOLDER else None
         self.logger, _ = get_logger(index, log_folder=log_folder, force=True)
 
     def snapshot(self, index, snapshot=None, recreate_repo=False):

biothings/hub/dataload/dumper.py

@@ -242,7 +242,7 @@ def post_dump(self, *args, **kwargs):
         pass
 
     def setup_log(self):
-        log_folder = os.path.join(btconfig.LOG_FOLDER, "dataload")
+        log_folder = os.path.join(btconfig.LOG_FOLDER, "dataload") if btconfig.LOG_FOLDER else None
         self.logger, self.logfile = get_logger("dump_%s" % self.src_name, log_folder=log_folder)
 
     def prepare(self, state={}):  # noqa: B006

biothings/hub/dataload/uploader.py

@@ -498,7 +498,7 @@ def prepare_src_dump(self):
         return src_dump
 
     def setup_log(self):
-        log_folder = os.path.join(config.LOG_FOLDER, "dataload")
+        log_folder = os.path.join(config.LOG_FOLDER, "dataload") if config.LOG_FOLDER else None
         return get_logger("upload_%s" % self.fullname, log_folder=log_folder)
 
     def __getattr__(self, attr):

biothings/hub/dataplugin/assistant.py

@@ -56,7 +56,7 @@ def __init__(self, plugin_name):
 
     def setup_log(self):
         """Setup and return a logger instance"""
-        log_folder = os.path.join(btconfig.LOG_FOLDER, "dataload")
+        log_folder = os.path.join(btconfig.LOG_FOLDER, "dataload") if btconfig.LOG_FOLDER else None
         self.logger, self.logfile = get_logger("loader_%s" % self.plugin_name, log_folder=log_folder)
 
     def get_plugin_obj(self):

biothings/hub/datarelease/publisher.py

@@ -70,7 +70,7 @@ def setup(self):
     def setup_log(self, build_name=None):
         log_folder = self.log_folder
         if build_name:
-            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name)
+            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name) if btconfig.LOG_FOLDER else None
         self.logger, self.logfile = get_logger(self.category, log_folder, force=True)
 
     def get_predicates(self):
@@ -1215,7 +1215,7 @@ def setup_log(self, build_name=None):
         name = RELEASEMANAGER_CATEGORY
         log_folder = self.log_folder
         if build_name:
-            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name)
+            log_folder = os.path.join(btconfig.LOG_FOLDER, "build", build_name) if btconfig.LOG_FOLDER else None
         self.logger, self.logfile = get_logger(name, log_folder=log_folder, force=True)
 
     def poll(self, state, func):

biothings/management/cli.py

@@ -6,6 +6,7 @@
 
 try:
     import typer
+    from rich.logging import RichHandler
 
     typer_avail = True
 except ImportError:
@@ -30,15 +31,16 @@
         no_args_is_help=True,
     )
 
-
-logger = logging.getLogger()
-logger.setLevel(level=logging.DEBUG)
+    logging.basicConfig(
+        level="INFO",
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, show_path=False)],
+    )
+    logger = logging.getLogger("cli")
 
 
-def main():
-    if not typer_avail:
-        logger.error('Error: "typer" package is required for CLI feature. Use "pip install typer[all]" to install.')
-        return
+def setup_config():
     working_dir = pathlib.Path().resolve()
     _config = DummyConfig("config")
     _config.HUB_DB_BACKEND = {
@@ -47,7 +49,8 @@ def main():
     }
     _config.DATA_SRC_DATABASE = ".data_src_database"
     _config.DATA_ARCHIVE_ROOT = ".biothings_hub/archive"
-    _config.LOG_FOLDER = ".biothings_hub/logs"
+    # _config.LOG_FOLDER = ".biothings_hub/logs"
+    _config.LOG_FOLDER = None  # disable file logging, only log to stdout
     _config.DATA_PLUGIN_FOLDER = f"{working_dir}"
     try:
         config_mod = importlib.import_module("config")
@@ -56,11 +59,20 @@ def main():
             if isinstance(value, ConfigurationError):
                 raise ConfigurationError("%s: %s" % (attr, str(value)))
             setattr(_config, attr, value)
-    except Exception:
-        logger.info("The config.py does not exists in the working directory, use default biothings.config")
+    except ModuleNotFoundError:
+        logger.debug("The config.py does not exists in the working directory, use default biothings.config")
+        pass
+
     sys.modules["config"] = _config
     sys.modules["biothings.config"] = _config
 
+
+def main():
+    if not typer_avail:
+        logger.error('"typer" package is required for CLI feature. Use "pip install typer[all]" to install.')
+        return
+
+    setup_config()
     from .dataplugin import app as dataplugin_app
     from .dataplugin_localhub import app as dataplugin_localhub_app
 

biothings/management/dataplugin.py

@@ -1,14 +1,16 @@
+# flake8: noqa: B008
 import os
 import pathlib
 from typing import Optional
 
 import typer
 from rich import print as rprint
 
-from biothings import config
-from biothings.utils.loggers import setup_default_log
+# from biothings import config
+from biothings.management.utils import get_logger
 
-logger = setup_default_log("dataplugin", config.LOG_FOLDER, "INFO")
+# logger = setup_default_log("dataplugin", config.LOG_FOLDER, "INFO")
+logger = get_logger("dataplugin")
 
 # To make sure biothings.config is initialized
 from . import utils
@@ -53,7 +55,7 @@ def listing(
 ):
     working_dir = pathlib.Path().resolve()
     plugin_name = working_dir.name
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     data_folder = os.path.join(working_dir, ".biothings_hub", "data_folder")
     if dump:
@@ -81,7 +83,7 @@ def clean_data(
     ),
 ):
     working_dir = pathlib.Path().resolve()
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     if dump:
         utils.do_clean_dumped_files(working_dir)
@@ -105,7 +107,7 @@ def dump_data(verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbo
     if verbose:
         logger.setLevel("DEBUG")
     working_dir = pathlib.Path().resolve()
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     manifest = utils.get_manifest_content(working_dir)
     to_dumps = utils.get_todump_list(manifest.get("dumper"))
@@ -138,7 +140,7 @@ def upload_source(
     if verbose:
         logger.setLevel("DEBUG")
     working_dir = pathlib.Path().resolve()
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     plugin_name = working_dir.name
     local_archive_dir = os.path.join(working_dir, ".biothings_hub")
@@ -168,7 +170,7 @@ def inspect(
     mode: Optional[str] = typer.Option(
         default="type,stats",
         help="""
-            The inspect mode or list of modes (comma separated) eg. "type,mapping".\n
+            The inspect mode or list of modes (comma separated), e.g. "type,mapping".\n
             Possible values are:\n
             - "type": explore documents and report strict data structure\n
             - "mapping": same as type but also perform test on data so guess best mapping\n
@@ -200,7 +202,7 @@ def inspect(
     if verbose:
         logger.setLevel("DEBUG")
     working_dir = pathlib.Path().resolve()
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     table_space = utils.get_uploaders(working_dir)
     if sub_source_name and sub_source_name not in table_space:
@@ -245,7 +247,7 @@ def serve(
     if verbose:
         logger.setLevel("DEBUG")
     working_dir = pathlib.Path().resolve()
-    if not utils.is_valid_working_directory(working_dir):
+    if not utils.is_valid_working_directory(working_dir, logger=logger):
         return exit(1)
     table_space = utils.get_uploaders(working_dir)
     data_plugin_name = working_dir.name

biothings/management/dataplugin_localhub.py

@@ -1,3 +1,4 @@
+# flake8: noqa: B008
 import os
 import pathlib
 from shutil import copytree
@@ -7,14 +8,7 @@
 import typer
 from rich import print as rprint
 
-from biothings.hub.dataload.dumper import DumperManager
-from biothings.hub.dataload.uploader import UploaderManager, upload_worker
-from biothings.hub.dataplugin.assistant import LocalAssistant
-from biothings.hub.dataplugin.manager import DataPluginManager
-from biothings.utils.hub_db import get_data_plugin
-from biothings.utils.loggers import get_logger
-
-from . import utils
+from biothings.management import utils
 
 short_help = "[green]Test multiple data plugins in a local minimal hub without any external databases.[/green]"
 long_help = (
@@ -73,6 +67,12 @@ def create_data_plugin(
 
 
 def load_plugin(plugin_name):
+    from biothings.hub.dataload.dumper import DumperManager
+    from biothings.hub.dataload.uploader import UploaderManager
+    from biothings.hub.dataplugin.assistant import LocalAssistant
+    from biothings.hub.dataplugin.manager import DataPluginManager
+    from biothings.utils.hub_db import get_data_plugin
+
     plugin_manager = DataPluginManager(job_manager=None)
     dmanager = DumperManager(job_manager=None)
     upload_manager = UploaderManager(job_manager=None)
@@ -123,6 +123,9 @@ def dump_and_upload(
     #     False, "--parallelizer", help="Using parallelizer or not? Default: No"
     # ),
 ):
+    from biothings.hub.dataload.uploader import upload_worker
+    from biothings.utils.hub_db import get_data_plugin
+
     working_dir = pathlib.Path().resolve()
     valid_names = [f.name for f in os.scandir(working_dir) if f.is_dir() and not f.name.startswith(".")]
     if not plugin_name or plugin_name not in valid_names:
@@ -186,11 +189,11 @@ def inspect(
         help="Your sub source name",
     ),
     mode: Optional[str] = typer.Option(
-        "mapping,type,stats",
+        "type,stats",
         "--mode",
         "-m",
         help="""
-            The inspect mode or list of modes (comma separated) eg. "type,mapping".\n
+            The inspect mode or list of modes (comma separated), e.g. "type,mapping".\n
             Possible values are:\n
             - "type": explore documents and report strict data structure\n
             - "mapping": same as type but also perform test on data so guess best mapping\n
@@ -225,7 +228,7 @@ def inspect(
         rprint("[red]Please provide your data plugin name! [/red]")
         rprint("Choose from:\n    " + "\n    ".join(valid_names))
         return exit(1)
-    logger, logfile = get_logger("inspect")
+    logger = utils.get_logger("inspect")
     if not limit:
         limit = None
     logger.info(f"Inspect Data plugin {plugin_name} with sub-source name: {sub_source_name} mode: {mode} limit {limit}")
@@ -272,6 +275,7 @@ def clean_data(
         help="Delete all dumped files and drop uploaded sources tables",
     ),
 ):
+    logger = utils.get_logger("clean")
     working_dir = pathlib.Path().resolve()
     valid_names = [f.name for f in os.scandir(working_dir) if f.is_dir() and not f.name.startswith(".")]
     if not plugin_name or plugin_name not in valid_names:
@@ -284,7 +288,7 @@ def clean_data(
     dumper.prepare()
     dumper.create_todump_list(force=True)
     data_plugin_dir = pathlib.Path(f"{working_dir}/{plugin_name}")
-    if not utils.is_valid_working_directory(data_plugin_dir):
+    if not utils.is_valid_working_directory(data_plugin_dir, logger=logger):
         return exit(1)
     if dump:
         utils.do_clean_dumped_files(pathlib.Path(dumper.new_data_folder), from_hub=True)

biothings/management/utils.py

@@ -1,6 +1,7 @@
 import asyncio
 import importlib
 import json
+import logging
 import math
 import os
 import pathlib
@@ -17,6 +18,7 @@
 from orjson import orjson
 from rich import box, print as rprint
 from rich.console import Console
+from rich.logging import RichHandler
 from rich.panel import Panel
 from rich.table import Table
 
@@ -28,6 +30,17 @@
 from biothings.utils.workers import upload_worker
 
 
+def get_logger(name):
+    logging.basicConfig(
+        level="INFO",
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, show_path=False)],
+    )
+    logger = logging.getLogger("cli")
+    return logger
+
+
 def get_uploaded_collections(src_db, uploaders):
     uploaded_sources = []
     archived_sources = []
@@ -550,10 +563,12 @@ def show_uploaded_sources(working_dir, plugin_name):
         )
 
 
-def is_valid_working_directory(working_dir):
+def is_valid_working_directory(working_dir, logger=None):
     if not os.path.isfile(f"{working_dir}/manifest.yaml") and not os.path.isfile(f"{working_dir}/manifest.json"):
-        rprint(
-            "[red]This command must be run inside a data plugin folder. Please go to a data plugin folder and try again! [/red]"
-        )
+        err = "[red]This command must be run inside a data plugin folder. Please go to a data plugin folder and try again! [/red]"
+        if logger:
+            logger.error(err, extra={"markup": True})
+        else:
+            rprint(err)
         return False
     return True