Merge pull request #667 from VijayKalmath/Save-ResultsSummaryinJson

Save Attack Summary Table in a Json

.gitignore

@@ -47,3 +47,4 @@ checkpoints/
 
 .vscode
 *.csv
+!tests/sample_outputs/csv_attack_log.csv

tests/sample_outputs/csv_attack_log.csv

@@ -0,0 +1,3 @@
+"ground_truth_output","num_queries","original_output","original_score","original_text","perturbed_output","perturbed_score","perturbed_text","result_type"
+1.0,28.0,1.0,0.09334743022918701,"lovingly photographed in the manner of a golden book sprung to [[life]] , stuart little 2 [[manages]] [[sweetness]] largely without stickiness .",0.0,0.6904040575027466,"lovingly photographed in the manner of a golden book sprung to [[ife]] , stuart little 2 [[manager]] [[seetness]] largely without stickiness .","Successful"
+1.0,16.0,1.0,0.009427368640899658,"[[consistently]] [[clever]] and [[suspenseful]] .",0.0,0.8219608664512634,"[[conisstently]] [[celver]] and [[Huspenseful]] .","Successful"

tests/sample_outputs/json_attack_summary.json

@@ -0,0 +1,13 @@
+{
+    "Attack Results": {
+        "Number of successful attacks:": 2,
+        "Number of failed attacks:": 0,
+        "Number of skipped attacks:": 0,
+        "Original accuracy:": 100.0,
+        "Accuracy under attack:": 0.0,
+        "Attack success rate:": 100.0,
+        "Average perturbed word %:": 45.0,
+        "Average num. words per input:": 12.0,
+        "Avg num queries:": 22.0
+    }
+}

tests/sample_outputs/txt_attack_log.txt

@@ -0,0 +1,21 @@
+--------------------------------------------- Result 1 ---------------------------------------------
+[[Positive (91%)]] --> [[Negative (69%)]]
+
+lovingly photographed in the manner of a golden book sprung to [[life]] , stuart little 2 [[manages]] [[sweetness]] largely without stickiness .
+
+lovingly photographed in the manner of a golden book sprung to [[ife]] , stuart little 2 [[manager]] [[seetness]] largely without stickiness .
+--------------------------------------------- Result 2 ---------------------------------------------
+[[Positive (99%)]] --> [[Negative (82%)]]
+
+[[consistently]] [[clever]] and [[suspenseful]] .
+
+[[conisstently]] [[celver]] and [[Huspenseful]] .
+Number of successful attacks: 2
+Number of failed attacks: 0
+Number of skipped attacks: 0
+Original accuracy: 100.0%
+Accuracy under attack: 0.0%
+Attack success rate: 100.0%
+Average perturbed word %: 45.0%
+Average num. words per input: 12.0
+Avg num queries: 22.0

tests/test_command_line/test_loggers.py

@@ -0,0 +1,101 @@
+import json
+import os
+
+from helpers import run_command_and_get_result
+import pytest
+
+DEBUG = False
+
+"""
+Attack command-line tests in the format (name, args, sample_output_file)
+"""
+
+"""
+ list_test_params data structure requires
+ 1) test name
+ 2) logger filetype - json/text/csv. # Future Work : Tests for Wandb and Visdom
+ 3) logger file name
+ 4) sample log file
+"""
+
+list_test_params = [
+    (
+        "json_summary_logger",
+        "json",
+        "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-summary-to-json attack_summary.json",
+        "attack_summary.json",
+        "tests/sample_outputs/json_attack_summary.json",
+    ),
+    (
+        "txt_logger",
+        "txt",
+        "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-to-txt attack_log.txt",
+        "attack_log.txt",
+        "tests/sample_outputs/txt_attack_log.txt",
+    ),
+    # Removing CSV Logging Test for time-being , will redo CSV test in separate PR.
+    #     (
+    #         "csv_logger",
+    #         "csv",
+    #         "textattack attack --recipe deepwordbug --model lstm-mr --num-examples 2 --log-to-csv attack_log.csv",
+    #         "attack_log.csv",
+    #         "tests/sample_outputs/csv_attack_log.csv",
+    #     ),
+]
+
+
+@pytest.mark.parametrize(
+    "name, filetype, command, test_log_file, sample_log_file", list_test_params
+)
+def test_logger(name, filetype, command, test_log_file, sample_log_file):
+    # Run command and validate outputs.
+    result = run_command_and_get_result(command)
+
+    assert result.stdout is not None
+    assert result.stderr is not None
+    assert result.returncode == 0
+    assert os.path.exists(test_log_file), f"{test_log_file} did not get generated"
+
+    if filetype == "json":
+        with open(sample_log_file) as f:
+            desired_dictionary = json.load(f)
+
+        with open(test_log_file) as f:
+            test_dictionary = json.load(f)
+
+        assert (
+            desired_dictionary == test_dictionary
+        ), f"{filetype} file {test_log_file} differs from {sample_log_file}"
+
+    elif filetype == "txt":
+        assert (
+            os.system(f"diff {test_log_file} {sample_log_file}") == 0
+        ), f"{filetype} file {test_log_file} differs from {sample_log_file}"
+
+    elif filetype == "csv":
+        import pandas as pd
+
+        # Convert them into dataframes and compare.
+        test_df = pd.read_csv(test_log_file)
+        sample_df = pd.read_csv(sample_log_file)
+        try:
+            test_df = test_df[sorted(list(test_df.columns.values))]
+            sample_df = sample_df[sorted(list(test_df.columns.values))]
+
+            for c in test_df.columns:
+                if test_df[c].dtype == int:
+                    test_df[c] = test_df[c].astype(float)
+
+                if sample_df[c].dtype == int:
+                    sample_df[c] = sample_df[c].astype(float)
+        except KeyError:
+            assert (
+                False
+            ), f"{filetype} file {test_log_file} differs from {sample_log_file}"
+
+        assert sample_df.equals(
+            test_df
+        ), f"{filetype} file {test_log_file} differs from {sample_log_file}"
+
+    # cleanup
+    os.remove(test_log_file)

textattack/attack_args.py

@@ -200,6 +200,7 @@ class AttackArgs:
     num_workers_per_device: int = 1
     log_to_txt: str = None
     log_to_csv: str = None
+    log_summary_to_json: str = None
     csv_coloring_style: str = "file"
     log_to_visdom: dict = None
     log_to_wandb: dict = None
@@ -326,6 +327,15 @@ def _add_parser_args(cls, parser):
             help="Path to which to save attack logs as a CSV file. Set this argument if you want to save CSV logs. "
             "If the last part of the path ends with `.csv` extension, the path is assumed to path for output file.",
         )
+        parser.add_argument(
+            "--log-summary-to-json",
+            nargs="?",
+            default=default_obj.log_summary_to_json,
+            const="",
+            type=str,
+            help="Path to which to save attack summary as a JSON file. Set this argument if you want to save attack results summary in a JSON. "
+            "If the last part of the path ends with `.json` extension, the path is assumed to path for output file.",
+        )
         parser.add_argument(
             "--csv-coloring-style",
             default=default_obj.csv_coloring_style,
@@ -418,6 +428,22 @@ def create_loggers_from_args(cls, args):
             )
             attack_log_manager.add_output_csv(csv_file_path, color_method)
 
+        # if '--log-summary-to-json' specified with arguments
+        if args.log_summary_to_json is not None:
+            if args.log_summary_to_json.lower().endswith(".json"):
+                summary_json_file_path = args.log_summary_to_json
+            else:
+                summary_json_file_path = os.path.join(
+                    args.log_summary_to_json, f"{timestamp}-attack_summary_log.json"
+                )
+
+            dir_path = os.path.dirname(summary_json_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(os.path.dirname(summary_json_file_path))
+
+            attack_log_manager.add_output_summary_json(summary_json_file_path)
+
         # Visdom
         if args.log_to_visdom is not None:
             attack_log_manager.enable_visdom(**args.log_to_visdom)

textattack/loggers/__init__.py

@@ -9,6 +9,7 @@
 from .logger import Logger
 from .visdom_logger import VisdomLogger
 from .weights_and_biases_logger import WeightsAndBiasesLogger
+from .json_summary_logger import JsonSummaryLogger
 
 # AttackLogManager must be imported last, since it imports the other loggers.
 from .attack_log_manager import AttackLogManager

textattack/loggers/attack_log_manager.py

@@ -10,7 +10,13 @@
 )
 from textattack.metrics.quality_metrics import Perplexity, USEMetric
 
-from . import CSVLogger, FileLogger, VisdomLogger, WeightsAndBiasesLogger
+from . import (
+    CSVLogger,
+    FileLogger,
+    JsonSummaryLogger,
+    VisdomLogger,
+    WeightsAndBiasesLogger,
+)
 
 
 class AttackLogManager:
@@ -39,6 +45,9 @@ def add_output_file(self, filename, color_method):
     def add_output_csv(self, filename, color_method):
         self.loggers.append(CSVLogger(filename=filename, color_method=color_method))
 
+    def add_output_summary_json(self, filename):
+        self.loggers.append(JsonSummaryLogger(filename=filename))
+
     def log_result(self, result):
         """Logs an ``AttackResult`` on each of `self.loggers`."""
         self.results.append(result)

textattack/loggers/json_summary_logger.py

@@ -0,0 +1,45 @@
+"""
+Attack Summary Results Logs to Json
+========================
+"""
+
+import json
+
+from textattack.shared import logger
+
+from .logger import Logger
+
+
+class JsonSummaryLogger(Logger):
+    def __init__(self, filename="results_summary.json"):
+        logger.info(f"Logging Summary to JSON at path {filename}")
+        self.filename = filename
+        self.json_dictionary = {}
+        self._flushed = True
+
+    def log_summary_rows(self, rows, title, window_id):
+        self.json_dictionary[title] = {}
+        for i in range(len(rows)):
+            row = rows[i]
+            if isinstance(row[1], str):
+                try:
+                    row[1] = row[1].replace("%", "")
+                    row[1] = float(row[1])
+                except ValueError:
+                    raise ValueError(
+                        f'Unable to convert row value "{row[1]}" for Attack Result "{row[0]}" into float'
+                    )
+
+        for metric, summary in rows:
+            self.json_dictionary[title][metric] = summary
+
+        self._flushed = False
+
+    def flush(self):
+        with open(self.filename, "w") as f:
+            json.dump(self.json_dictionary, f, indent=4)
+
+        self._flushed = True
+
+    def close(self):
+        super().close()

textattack/loggers/logger.py

@@ -13,7 +13,7 @@ class Logger(ABC):
     def __init__(self):
         pass
 
-    def log_attack_result(self, result, examples_completed):
+    def log_attack_result(self, result, examples_completed=None):
         pass
 
     def log_summary_rows(self, rows, title, window_id):