From f870f0e394d2e692be9c875750492d93af020873 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sun, 4 Feb 2024 23:24:57 +0800 Subject: [PATCH] CLI migration --- src/thread/cli.py | 9 ++ src/thread/cli/__init__.py | 18 --- src/thread/cli/base.py | 117 --------------- src/thread/cli/process.py | 229 ----------------------------- src/thread/cli/utils.py | 48 ------ src/thread/utils/__init__.py | 1 - src/thread/utils/logging_config.py | 31 ---- 7 files changed, 9 insertions(+), 444 deletions(-) create mode 100644 src/thread/cli.py delete mode 100644 src/thread/cli/__init__.py delete mode 100644 src/thread/cli/base.py delete mode 100644 src/thread/cli/process.py delete mode 100644 src/thread/cli/utils.py delete mode 100644 src/thread/utils/logging_config.py diff --git a/src/thread/cli.py b/src/thread/cli.py new file mode 100644 index 0000000..63847a5 --- /dev/null +++ b/src/thread/cli.py @@ -0,0 +1,9 @@ + +try: + import importlib + thread_cli = importlib.import_module('thread-cli') + app = thread_cli.app +except ModuleNotFoundError: + def app(prog_name = 'thread'): + print('thread-cli not found, please install it with `pip install thread-cli`') + exit(1) \ No newline at end of file diff --git a/src/thread/cli/__init__.py b/src/thread/cli/__init__.py deleted file mode 100644 index 2b81df7..0000000 --- a/src/thread/cli/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Import and config CLI commands -""" - -__version__ = '0.1.3' -from ..utils.logging_config import logging, ColorLogger -logging.setLoggerClass(ColorLogger) - - -# Import # -from .base import cli_base as app -from .process import process as process_cli - -app.command( - name = 'process', - no_args_is_help = True, - context_settings = {'allow_extra_args': True} -)(process_cli) diff --git a/src/thread/cli/base.py b/src/thread/cli/base.py deleted file mode 100644 index 768cd82..0000000 --- a/src/thread/cli/base.py +++ /dev/null @@ -1,117 +0,0 @@ -import typer -import logging - -from . import __version__ -from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor -logger = logging.getLogger('base') - - -cli_base = typer.Typer( - no_args_is_help = True, - rich_markup_mode = 'rich', - context_settings = { - 'help_option_names': ['-h', '--help', 'help'] - } -) - - -def version_callback(value: bool): - if value: - typer.echo(f'v{__version__}') - raise typer.Exit() - - -@cli_base.callback(invoke_without_command = True) -def callback( - version: bool = typer.Option( - None, '--version', - callback = version_callback, - help = 'Get the current installed version', - is_eager = True - ), - - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption -): - """ - [b]Thread CLI[/b]\b\n - [white]Use thread from the terminal![/white] - - [blue][u] [/u][/blue] - - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) - - - -# Help and Others -@cli_base.command(rich_help_panel = 'Help and Others') -def help(): - """Get [yellow]help[/yellow] from the community. :question:""" - typer.echo('Feel free to search for or ask questions here!') - try: - logger.info('Attempting to open in web browser...') - - import webbrowser - webbrowser.open( - 'https://github.com/python-thread/thread/issues', - new = 2 - ) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') - - - -@cli_base.command(rich_help_panel = 'Help and Others') -def docs(): - """View our [yellow]documentation.[/yellow] :book:""" - typer.echo('Thanks for using Thread, here is our documentation!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser - webbrowser.open( - 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', - new = 2 - ) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/blob/main/docs/command-line.md') - - - -@cli_base.command(rich_help_panel = 'Help and Others') -def report(): - """[yellow]Report[/yellow] an issue. :bug:""" - typer.echo('Sorry you run into an issue, report it here!') - try: - logger.info('Attempting to open in web browser...') - import webbrowser - webbrowser.open( - 'https://github.com/python-thread/thread/issues', - new = 2 - ) - typer.echo('Opening in web browser!') - - except Exception as e: - logger.warn('Failed to open web browser') - logger.debug(f'{e}') - typer.echo('https://github.com/python-thread/thread/issues') - - - -# Utils and Configs -@cli_base.command(rich_help_panel = 'Utils and Configs') -def config(configuration: str): - """ - [blue]Configure[/blue] the system. :wrench: - """ - typer.echo('Coming soon!') diff --git a/src/thread/cli/process.py b/src/thread/cli/process.py deleted file mode 100644 index 30453b6..0000000 --- a/src/thread/cli/process.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Parallel Processing command""" - -import os -import time -import json -import inspect -import importlib - -import typer -import logging -from typing import Union - -from rich.live import Live -from rich.panel import Panel -from rich.console import Group -from rich.progress import Progress, TaskID, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn, TimeElapsedColumn -from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor, kwargs_processor - -logger = logging.getLogger('base') - - -def process( - func: str = typer.Argument(help = '[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]'), - dataset: str = typer.Argument(help = '[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]'), - - args: list[str] = typer.Option([], '--arg', '-a', help = '[blue]Arguments[/blue] passed to each thread'), - kargs: list[str] = typer.Option([], '--kwarg', '-kw', help = '[blue]Key-Value arguments[/blue] passed to each thread'), - threads: int = typer.Option(8, '--threads', '-t', help = 'Maximum number of [blue]threads[/blue] (will scale down based on dataset size)'), - - daemon: bool = typer.Option(False, '--daemon', '-d', help = 'Threads to run in [blue]daemon[/blue] mode'), - graceful_exit: bool = typer.Option(True, '--graceful-exit', '-ge', is_flag = True, help = 'Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)'), - output: str = typer.Option('./output.json', '--output', '-o', help = '[blue]Output[/blue] file location'), - fileout: bool = typer.Option(True, '--fileout', is_flag = True, help = 'Whether to [blue]write[/blue] output to a file'), - stdout: bool = typer.Option(False, '--stdout', is_flag = True, help = 'Whether to [blue]print[/blue] the output'), - - debug: bool = DebugOption, - verbose: bool = VerboseOption, - quiet: bool = QuietOption -): - """ - [bold]Utilise parallel processing on a dataset[/bold] - - \b\n - [bold white]:glowing_star: Important[/bold white] - Args and Kwargs can be parsed by adding multiple -a or -kw - - [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 - [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] - [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] - - [blue][u] [/u][/blue] - - Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] - """ - verbose_args_processor(debug, verbose, quiet) - kwargs = kwargs_processor(kargs) - logger.debug('Processed kwargs: %s' % kwargs) - - - # Verify output - if not fileout and not stdout: - raise typer.BadParameter('No output method specified') - - if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): - raise typer.BadParameter('Output file directory does not exist') - - - - - # Loading function - f = None - try: - logger.info('Attempted to interpret function') - f = eval(func) # I know eval is bad practice, but I have yet to find a safer replacement - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Invalid function') - except Exception: - logger.info('Failed to interpret function') - - if not f: - try: - logger.info('Attempting to fetch function file') - - fPath, fName = func.split(':') - f = importlib.import_module(fPath).__dict__[fName] - logger.debug('Evaluated function: %s' % f) - - if not inspect.isfunction(f): - logger.info('Not a function') - raise Exception('Not a function') - except Exception as e: - logger.warning('Failed to fetch function') - raise typer.BadParameter('Failed to fetch function') from e - - - - - # Loading dataset - ds: Union[list, tuple, set, None] = None - try: - logger.info('Attempting to interpret dataset') - ds = eval(dataset) - logger.debug('Evaluated dataset: %s' % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds)) - - if not isinstance(ds, (list, tuple, set)): - logger.info('Invalid dataset literal') - ds = None - - except Exception: - logger.info('Failed to interpret dataset') - - if not ds: - try: - logger.info('Attempting to fetch data file') - if not os.path.isfile(dataset): - logger.info('Invalid file path') - raise Exception('Invalid file path') - - with open(dataset, 'r') as a: - ds = [ i.endswith('\n') and i[:-2] for i in a.readlines() ] - except Exception as e: - logger.warning('Failed to read dataset') - raise typer.BadParameter('Failed to read dataset') from e - - logger.info('Interpreted dataset') - - - # Setup - logger.debug('Importing module') - from ..thread import Settings, ParallelProcessing - logger.info('Spawning threads... [Expected: {tcount} threads]'.format(tcount=min(len(ds), threads))) - - Settings.set_graceful_exit(graceful_exit) - newProcess = ParallelProcessing( - function = f, - dataset = list(ds), - args = args, - kwargs = kwargs, - daemon = daemon, - max_threads = threads - ) - - logger.info('Created parallel process') - logger.info('Starting parallel process') - - start_t = time.perf_counter() - newProcess.start() - - logger.info('Started parallel processes') - typer.echo('Waiting for parallel processes to complete, this may take a while...') - - - # Progress bar :D - threadCount = len(newProcess._threads) - - thread_progress = Progress( - SpinnerColumn(), - TextColumn('{task.description}'), - '•', - TimeRemainingColumn(), - BarColumn(bar_width = 80), - TextColumn('{task.percentage:>3.1f}%') - ) - overall_progress = Progress( - TimeElapsedColumn(), - BarColumn(bar_width = 110), - TextColumn('{task.description}') - ) - - workerjobs: list[TaskID] = [ - thread_progress.add_task( - f'[bold blue][T {threadNum}]', - total = 100 - ) - for threadNum in range(threadCount) - ] - overalljob = overall_progress.add_task('(0 of ?)', total = 100) - - - with Live( - Group( - Panel(thread_progress), - overall_progress, - ), - refresh_per_second = 10 - ): - completed = 0 - while completed != threadCount: - i = 0 - completed = 0 - progressAvg = 0 - - for jobID in workerjobs: - jobProgress = newProcess._threads[i].progress - thread_progress.update(jobID, completed = round(jobProgress * 100, 2)) - if jobProgress == 1: - thread_progress.stop_task(jobID) - thread_progress.update(jobID, description = '[bold green]Completed') - completed += 1 - - progressAvg += jobProgress - i += 1 - - # Update overall - overall_progress.update( - overalljob, - description = f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', - completed = round((progressAvg / threadCount) * 100, 2) - ) - time.sleep(0.1) - - - result = newProcess.get_return_values() - - typer.echo(f'Completed in {(time.perf_counter() - start_t):.5f}s') - if fileout: - typer.echo(f'Writing to {output}') - try: - with open(output, 'w') as f: - json.dump(result, f, indent = 2) - logger.info('Wrote to file') - except Exception as e: - logger.error('Failed to write to file') - logger.debug(str(e)) - - if stdout: - typer.echo(result) diff --git a/src/thread/cli/utils.py b/src/thread/cli/utils.py deleted file mode 100644 index 035c80f..0000000 --- a/src/thread/cli/utils.py +++ /dev/null @@ -1,48 +0,0 @@ -# Verbose Command Processor # -import typer -import logging - - -# Verbose Options # -DebugOption = typer.Option( - False, '--debug', - help = 'Set verbosity level to [blue]DEBUG[/blue]', - is_flag = True -) -VerboseOption = typer.Option( - False, '--verbose', '-v', - help = 'Set verbosity level to [green]INFO[/green]', - is_flag = True -) -QuietOption = typer.Option( - False, '--quiet', '-q', - help = 'Set verbosity level to [red]ERROR[/red]', - is_flag = True -) - - -# Helper functions # - - -# Processors # -def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): - """Handles setting and raising exceptions for verbose""" - if verbose and quiet: - raise typer.BadParameter('--quiet cannot be used with --verbose') - - if verbose and debug: - raise typer.BadParameter('--debug cannot be used with --verbose') - - logging.getLogger('base').setLevel(( - (debug and logging.DEBUG) or - (verbose and logging.INFO) or - logging.ERROR - )) - -def kwargs_processor(arguments: list[str]) -> dict[str, str]: - """Processes arguments into kwargs""" - return { - kwarg[0]: kwarg[1] - for i in arguments - if (kwarg := i.split('=')) - } diff --git a/src/thread/utils/__init__.py b/src/thread/utils/__init__.py index 16e9e45..1063cca 100644 --- a/src/thread/utils/__init__.py +++ b/src/thread/utils/__init__.py @@ -2,7 +2,6 @@ Export utility functions and libraries """ -from .logging_config import ColorLogger from .config import Settings from . import ( diff --git a/src/thread/utils/logging_config.py b/src/thread/utils/logging_config.py deleted file mode 100644 index 647d396..0000000 --- a/src/thread/utils/logging_config.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging -from colorama import init, Fore, Style - -init(autoreset = True) - - -# Stdout color config # -class ColorFormatter(logging.Formatter): - COLORS = { - 'DEBUG' : Fore.BLUE, - 'INFO' : Fore.GREEN, - 'WARNING' : Fore.YELLOW, - 'ERROR' : Fore.RED, - 'CRITICAL': Fore.RED + Style.BRIGHT - } - - def format(self, record): - color = self.COLORS.get(record.levelname, '') - if color: - record.levelname = color + Style.BRIGHT + f'{record.levelname:<9}|' - record.msg = color + Fore.WHITE + Style.NORMAL + record.msg - return logging.Formatter.format(self, record) - - -class ColorLogger(logging.Logger): - def __init__(self, name): - logging.Logger.__init__(self, name, logging.DEBUG) - color_formatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) - console = logging.StreamHandler() - console.setFormatter(color_formatter) - self.addHandler(console)