feat: add unused command (#8)

docs/source/cli-commands.rst

@@ -13,13 +13,15 @@ This section describe the Conan built-in commands, like ``pdbstore add`` or ``pd
    commands/add
    commands/clean
    commands/del
+   commands/fetch
    commands/query
    commands/report
-   commands/fetch
+   commands/unused
 
 - :doc:`pdbstore add <commands/add>`: Add files to local symbol store
 - :doc:`pdbstore clean <commands/clean>`: Remove old transactions associated given some criteria
 - :doc:`pdbstore del <commands/del>`: Delete transaction from local symbol store
+- :doc:`pdbstore fetch <commands/fetch>`: Fetch symbol files from for a local symbol store
 - :doc:`pdbstore query <commands/query>`: Check if file(s) are indexed from local symbol store
 - :doc:`pdbstore report <commands/report>`: Generate report for a local symbol store
-- :doc:`pdbstore fetch <commands/fetch>`: Fetch symbol files from for a local symbol store
+- :doc:`pdbstore unused <commands/unused>`: Find all files not used since a specific date

docs/source/commands/fetch.rst

@@ -6,6 +6,9 @@ pdbstore fetch
 .. code-block:: text
 
     $ pdbstore fetch -h
+    usage: pdbstore fetch [-s DIRECTORY] [-r] [-O DIR] [-F] [-C PATH] [-S NAME] 
+                          [-L PATH] [-V [LEVEL]] [-f NAME] [-h] [FILE_OR_DIR ...]
+
     Fetch all files from a symbol store
 
     positional arguments:

docs/source/commands/query.rst

@@ -6,6 +6,9 @@ pdbstore query
 .. code-block:: text
 
     $ pdbstore query -h
+    usage: pdbstore fetch [-s DIRECTORY] [-r] [-F] [-C PATH] [-S NAME] [-L PATH]
+                          [-V [LEVEL]] [-f NAME] [-h] [FILE_OR_DIR ...]
+
     Check if file(s) are indexed on the server
 
     positional arguments:

docs/source/commands/unused.rst

@@ -0,0 +1,39 @@
+.. _commands_unused:
+
+pdbstore unused
+===============
+
+.. code-block:: text
+
+    $ pdbstore unused -h
+    usage: pdbstore unused [-s DIRECTORY] [-C PATH] [-S NAME] [-L PATH] 
+                           [-V [LEVEL]] [-f NAME] [-h] [DATE]
+
+    Find all files not used since a specific date
+
+    positional arguments:
+      DATE                  Date given YYYY-MM-DD format.
+
+    options:
+      -s DIRECTORY, --store-dir DIRECTORY
+                            Local root directory for the symbol store. [env var:        
+                            PDBSTORE_STORAGE_DIR]
+      -C PATH, --config-file PATH
+                            Configuration file to use. Can be used multiple times.      
+                            [env var: PDBSTORE_CFG]
+      -S NAME, --store NAME Which configuration section should be used. If not
+                            defined, the default will be used
+      -L PATH, --log-file PATH
+                            Send output to PATH instead of stderr.
+      -V [LEVEL], --verbosity [LEVEL]
+                            Level of detail of the output. Valid options from less      
+                            verbose to more verbose: -Vquiet, -Verror, -Vwarning,       
+                            -Vnotice, -Vstatus, -V or -Vverbose, -VV or -Vdebug, -VVV   
+                            or -vtrace
+      -f NAME, --format NAME
+                            Select the output format: json
+      -h, --help            show this help message and exit
+
+
+The ``pdbstore unused`` command will automatically determine all files that have not
+been used since the specified date by comparing it to the most recent access time.

pdbstore/cli/cli.py

@@ -178,7 +178,7 @@ def run(self, *args: Any) -> ExitCode:
             command.run(args[0][1:])
         except Exception as exc:
             if PDBStoreOutput.level_allowed(LEVEL_TRACE):
-                print(traceback.format_exc(), file=sys.stderr)
+                output.trace("\n".join(traceback.format_exception(exc)))
             raise exc
         return 0
 
@@ -195,13 +195,13 @@ def exception_exit_error(exception: Optional[BaseException]) -> ExitCode:
         if isinstance(exception, PDBInvalidSubCommandNameException):
             return ERROR_SUBCOMMAND_NAME
         if isinstance(exception, CommandLineError):
-            output.error(str(exception))
+            output.error(exception.message)
             return ERROR_UNEXPECTED
         if isinstance(exception, ConfigError):
-            output.error(exception)
+            output.error(exception.message)
             return ERROR_INVALID_CONFIGURATION
         if isinstance(exception, PDBStoreException):
-            output.error(exception)
+            output.error(exception.message)
             return ERROR_GENERAL
         if isinstance(exception, SystemExit):
             if exception.code != 0:

pdbstore/cli/command.py

@@ -67,7 +67,7 @@ def parse_args(  # type: ignore[override] # pylint: disable=arguments-differ
         except ConfigError as exc:
             if "--help" in (args or sys.argv) or "-h" in (args or sys.argv):
                 self.print_help()
-                raise PDBAbortExecution(0)  # pylint: disable=raise-missing-from
+                raise PDBAbortExecution(0) from exc
             raise exc
 
         args_dict = vars(options)

pdbstore/cli/commands/report.py

@@ -48,7 +48,7 @@ def report_html_formatter(report_dict: ReportDict) -> None:
     _report_render(report_dict, "html")
 
 
-@pdbstore_command(group="Usage")
+@pdbstore_command(group="Analysis")
 def report(
     parser: PDBStoreArgumentParser,  # pylint: disable=unused-argument
     *args: Any,  # pylint: disable=unused-argument

pdbstore/cli/commands/unused.py

@@ -0,0 +1,142 @@
+import json
+import os
+import time
+from pathlib import Path
+
+from pdbstore import util
+from pdbstore.cli.args import add_global_arguments, add_storage_arguments
+from pdbstore.cli.command import pdbstore_command, PDBStoreArgumentParser
+from pdbstore.exceptions import CommandLineError, PDBAbortExecution, PDBStoreException
+from pdbstore.io.output import cli_out_write, PDBStoreOutput
+from pdbstore.store import OpStatus, Store, Summary, TransactionType
+from pdbstore.typing import Any
+
+
+def unused_text_formatter(summary: Summary) -> None:
+    """Print output text from a Summary object as TEXT format"""
+    if len(summary.files) == 0:
+        return
+
+    input_len = 80
+    cli_out_write(f"{'Input File':<{input_len}s}{'Date':^10s}  Transaction ID")
+    for cur in summary.iterator():
+        files_list = sorted(cur.files, key=lambda k: k.get("date", "N/A"))
+        for file_entry in files_list:
+            file_path = file_entry.get("path", "")
+            file_date = file_entry.get("date", "N/A")
+            transaction_id = file_entry.get("transaction_id", "N/A")
+            status: OpStatus = OpStatus.from_str(
+                file_entry.get("status", OpStatus.SKIPPED)
+            )
+            error_msg = file_entry.get("error")
+
+            file_path = util.abbreviate(file_path, 80)
+
+            if status == OpStatus.SUCCESS:
+                cli_out_write(
+                    f"{str(file_path):<{input_len}s}{file_date:^10s}  {transaction_id}"
+                )
+            else:
+                cli_out_write(
+                    f"{str(file_path):<{input_len}s}{'N/A':^10s}  {error_msg or 'File not found'}"
+                )
+
+    total = summary.failed(True)
+    if total > 0:
+        raise PDBAbortExecution(total)
+
+
+def unused_json_formatter(summary: Summary) -> None:
+    """Print output text from a Summary object as JSON format"""
+    if len(summary.files) == 0:
+        cli_out_write("[]")
+        return
+
+    out = []
+    for cur in summary.iterator():
+        dct = {
+            "status": cur.status.value,
+            "success": cur.success(False),
+            "failure": cur.failed(True),
+            "skip": cur.skipped(True),
+            "files": sorted(cur.files, key=lambda k: k.get("date", "N/A")),
+            "message": cur.error_msg or "",
+        }
+        out.append(dct)
+
+    cli_out_write(json.dumps(out, indent=4))
+
+    total = summary.failed(True)
+    if total > 0:
+        raise PDBAbortExecution(total)
+
+
+@pdbstore_command(
+    group="Analysis",
+    formatters={"text": unused_text_formatter, "json": unused_json_formatter},
+)
+def unused(parser: PDBStoreArgumentParser, *args: Any) -> Any:
+    """
+    Find all files not used since a specific date
+    """
+    add_storage_arguments(parser)
+
+    parser.add_argument(
+        "date",
+        metavar="DATE",
+        nargs="?",
+        type=str,
+        help="""Date given YYYY-MM-DD format.""",
+    )
+
+    add_global_arguments(parser)
+
+    opts = parser.parse_args(*args)
+
+    output = PDBStoreOutput()
+
+    # Check input configuration and arguments
+    store_dir = opts.store_dir
+    if not store_dir:
+        raise CommandLineError("no symbol store directory given")
+
+    input_date = opts.date
+    if not input_date:
+        raise CommandLineError("no date given")
+
+    store = Store(store_dir)
+
+    try:
+        input_date = time.strptime(input_date, "%Y-%m-%d")
+    except ValueError as vexc:
+        raise CommandLineError(f"'{input_date}' invalid date given") from vexc
+
+    output.verbose(f"Search files not used since {input_date}")
+    input_date = time.mktime(input_date)
+
+    # Check for each file is present to the specified store or not.
+    summary = Summary(None, OpStatus.SUCCESS, TransactionType.UNUSED)
+
+    for transaction, entry in store.iterator(lambda x: not x.is_deleted()):
+        try:
+            output.verbose(f"checking {entry.rel_path} ...")
+            file_path: Path = entry.stored_path
+            file_stat: os.stat_result = file_path.stat()
+            if file_stat.st_atime < input_date:
+                dct = summary.add_file(entry.rel_path, OpStatus.SUCCESS)
+                dct["date"] = time.strftime(
+                    "%Y-%m-%d", time.localtime(file_stat.st_atime)
+                )
+                dct["transaction_id"] = transaction.id
+        except PDBStoreException as exp:  # pragma: no cover
+            summary.add_file(
+                util.path_to_str(entry.rel_path), OpStatus.FAILED, "ex:" + str(exp)
+            )
+        except Exception as exc:  # pylint: disable=broad-except # pragma: no cover
+            summary.add_file(
+                util.path_to_str(entry.rel_path), OpStatus.FAILED, str(exc)
+            )
+            output.error(exc)
+            output.error("unexpected error when checking {file_path} file usage")
+
+    return summary

pdbstore/exceptions.py

@@ -9,6 +9,7 @@ class PDBStoreException(Exception):
 
     def __init__(self, message: str) -> None:
         Exception.__init__(self, message)
+        self.message = message
 
 
 #

pdbstore/io/output.py

@@ -303,13 +303,9 @@ def error(self, msg: Union[BaseException, str, Dict[str, str]]) -> "PDBStoreOutp
         Prints an error message.
         """
         if self._pdbstore_output_level <= LEVEL_ERROR:
-            if isinstance(msg, BaseException):
-                sio = io.StringIO()
-                traceback.print_exception(
-                    type(msg), value=msg, tb=msg.__traceback__, file=sio
-                )
-                exc_msg: str = sio.getvalue()
-                sio.close()
+            if isinstance(msg, BaseException): # pragma: no cover
+                lines = traceback.format_exception(msg)
+                exc_msg = ("\n".join(lines)).replace("\n", "\n       ")
                 self._write_message(f"ERROR: {exc_msg}", Color.RED)
             else:
                 self._write_message(f"ERROR: {msg}", Color.RED)

pdbstore/store/entry.py

@@ -47,6 +47,16 @@ def file_path(self) -> Path:
         """
         return self.source_file
 
+    @property
+    def rel_path(self) -> Path:
+        """Retrieve the relative path to the stored file
+
+        :return: Relative path name to the stored file
+        """
+        if not self.compressed:
+            return Path(self.file_name, self.file_hash, self.file_name)
+        return Path(self.file_name, self.file_hash, (str(self.file_name)[:-1] + "_"))
+
     @property
     def stored_path(self) -> Path:
         """Retrieve the full path to the stored file

pdbstore/store/store.py

@@ -11,7 +11,7 @@
 from pdbstore.store.transaction import Transaction
 from pdbstore.store.transaction_type import TransactionType
 from pdbstore.store.transactions import Transactions
-from pdbstore.typing import List, Optional, PathLike, Tuple
+from pdbstore.typing import Callable, Generator, List, Optional, PathLike, Tuple
 
 __all__ = ["Store"]
 
@@ -341,3 +341,15 @@ def remove_old_versions(
             next_summary = trans_summary
 
         return summary or Summary()
+
+    def iterator(
+        self, filter_cb: Optional[Callable[[Transaction], bool]] = None
+    ) -> Generator[Tuple[Transaction, TransactionEntry], None, None]:
+        """Iterate over all transactions and file entries.
+
+        :param product: Optional callback function to filter transactions.
+        """
+        for transaction in self.transactions.transactions.values():
+            if not filter_cb or filter_cb(transaction):
+                for entry in transaction.entries:
+                    yield (transaction, entry)

pdbstore/store/summary.py

@@ -97,7 +97,7 @@ def skipped(self, full: bool = False) -> int:
     def referenced(self, full: bool = False) -> int:
         """Retrieve the total of modified references."""
         return self._references + (
-            self._linked.skipped() if (full and self._linked) else 0
+            self._linked.referenced() if (full and self._linked) else 0
         )
 
     def count(self, success_only: bool = False) -> int:

pdbstore/store/transaction_type.py

@@ -17,3 +17,5 @@ class TransactionType(Enum):
     """ Query files from symbol store"""
     FETCH = "fetch"
     """ Search and extract files from symbol store"""
+    UNUSED = "unused"
+    """ Find all files not used since a specific date from symbol store"""

pdbstore/store/transactions.py

@@ -79,7 +79,7 @@ def _parse(self) -> Dict[str, Transaction]:
             ReadFileError: Failed to read server file
         """
         if not self._server_file_exists():
-            PDBStoreOutput().warning(f"{str(self.store.server_file_path)} not found")
+            PDBStoreOutput().verbose(f"{str(self.store.server_file_path)} not found")
             return {}
 
         transactions = {}