Implement the MUNI/33/0769/2022 R&D project

pv211_utils/arqmath/eval.py

@@ -15,6 +15,8 @@ class ArqmathEvaluation(EvaluationBase):
         The information retrieval system.
     judgements : set of ArqmathJudgementBase
         Pairs of queries and relevant documents.
+    k : int, optional
+        Parameter defining evaluation depth. Default is 10.
     leaderboard : ArqmathLeaderboard or None, optional
         A leaderboard to which we may later submit evaluation results.
         If None, then evaluation results will not be submitted. Default is None.
@@ -24,12 +26,11 @@ class ArqmathEvaluation(EvaluationBase):
     num_workers : int or None, optional
         The number of processes used to compute the mean average precision.
         If None, all available CPUs will be used. Default is 1.
-
     """
     def __init__(self, system: ArqmathIRSystemBase, judgements: Set[ArqmathJudgementBase],
-                 leaderboard: Optional[ArqmathLeaderboard] = None,
+                 k: Optional[int] = 10, leaderboard: Optional[ArqmathLeaderboard] = None,
                  author_name: Optional[str] = None, num_workers: Optional[int] = 1):
-        super().__init__(system, judgements, leaderboard, author_name, num_workers)
+        super().__init__(system, judgements, k, leaderboard, author_name, num_workers)
 
     def _get_minimum_mean_average_precision(self) -> float:
         return 0.012

pv211_utils/beir/entities.py

@@ -0,0 +1,86 @@
+from typing import Any, Tuple, Optional, List, Set
+
+from pv211_utils.entities import DocumentBase, QueryBase
+
+
+class BeirDocumentBase(DocumentBase):
+    """A Generic document with just an id and a body.
+
+    Parameters
+    ----------
+    document_id : str
+        A unique identifier of the document.
+    body : Any
+        The text of the document.
+
+    """
+    def __init__(self, document_id: str, body: Any):
+        super().__init__(document_id, body)
+
+
+class BeirQueryBase(QueryBase):
+    """A Generic query with just an id and a body
+
+    Parameters
+    ----------
+    query_id : int
+        A unique identifier of the query.
+    body : Any
+        The text of the query.
+
+    """
+    def __init__(self, query_id: int, body: Any):
+        super().__init__(query_id, body)
+
+
+class RawBeirDataset:
+    """A generic BIER dataset with options
+
+    Parameters
+    ----------
+    name : str
+        A unique identifier of the dataset, select one name from the list of available datasets.
+    train : bool
+        A bool value that dictates if a train subset should be prepared.
+    dev : bool
+        A bool value that dictates if a dev subset should be prepared.
+    test : bool
+        A bool value that dictates if a test subset should be prepared.
+    train_alternative : str
+        An alternative in case the given dataset does not come with an available training subset.
+    dev_alternative : str
+        An alternative in case the given dataset does not come with an available dev subset.
+    test_alternative : str
+        An alternative in case the given dataset does not come with an available test subset.
+
+    """
+    def __init__(self, name: str,
+                 train: bool = False, dev: bool = False, test: bool = False,
+                 train_alternative: Optional[str] = None, dev_alternative: Optional[str] = None,
+                 test_alternative: Optional[str] = None):
+        self.name = name
+        self.train = train
+        self.dev = dev
+        self.test = test
+        self.train_alternative = train_alternative
+        self.dev_alternative = dev_alternative
+        self.test_alternative = test_alternative
+
+
+class RawBeirDatasets:
+    """A list of RawBeirDataset that will be loaded, prepared and merged into one for use in IR systems.
+
+    Parameters
+    ----------
+    download_location : str
+        A address where all the datasets will be downloaded. Access availability is required.
+    datasets : List[RawBeirDataset]
+        A list of RawBeirDataset values.
+    """
+    def __init__(self, datasets: List[RawBeirDataset], download_location: str):
+        self.download_location = download_location
+        self.datasets = datasets
+
+
+BeirJudgementBase = Tuple[BeirQueryBase, BeirDocumentBase]
+BeirJudgementsBase = Set[BeirJudgementBase]

pv211_utils/beir/eval.py

@@ -0,0 +1,34 @@
+from typing import Set, Optional
+
+from pv211_utils.eval import EvaluationBase
+from .entities import BeirJudgementBase
+from .irsystem import BeirIRSystemBase
+
+
+class BeirEvaluation(EvaluationBase):
+    """A Generic information retrieval system evaluation.
+
+    Parameters
+    ----------
+    system : BeirIRSystemBase
+        The information retrieval system.
+    judgements : set of BeirJudgementBase
+        Pairs of queries and relevant documents.
+    k : int, optional
+        Parameter defining evaluation depth. Default is 10.
+    leaderboard : BeirLeaderboard or None, optional
+        A leaderboard to which we may later submit evaluation results.
+        If None, then evaluation results will not be submitted. Default is None.
+    author_name : str or None, optional
+        The name of the author submitted to the leaderboard.
+        If None, then the result will not be submitted. Default is None.
+    num_workers : int or None, optional
+        The number of processes used to compute the mean average precision.
+        If None, all available CPUs will be used. Default is 1.
+    """
+    def __init__(self, system: BeirIRSystemBase, judgements: Set[BeirJudgementBase], k: Optional[int] = 10,
+                 leaderboard=None, author_name: Optional[str] = None, num_workers: Optional[int] = 1):
+        super().__init__(system, judgements, k, leaderboard, author_name, num_workers)
+
+    def _get_minimum_mean_average_precision(self) -> float:
+        return 0.1

pv211_utils/beir/irsystem.py

@@ -0,0 +1,11 @@
+import abc
+from typing import Iterable
+
+from ..irsystem import IRSystemBase
+from .entities import BeirQueryBase, BeirDocumentBase
+
+
+class BeirIRSystemBase(IRSystemBase):
+    @abc.abstractmethod
+    def search(self, query: BeirQueryBase) -> Iterable[BeirDocumentBase]:
+        pass

pv211_utils/beir/leaderboard.py

@@ -0,0 +1,27 @@
+import datetime
+import pkg_resources
+from typing import Optional
+
+from ..leaderboard import GoogleSpreadsheetLeaderboardBase
+
+
+class BeirLeaderboard(GoogleSpreadsheetLeaderboardBase):
+    def get_competition_start(self) -> datetime.date:
+        return datetime.date(2022, 8, 18)
+
+    def get_competition_end(self) -> datetime.date:
+        return datetime.date(2022, 12, 30)
+
+    def _get_key_path(self) -> str:
+        key_path = pkg_resources.resource_filename('pv211_utils', 'data/pv211-leaderboard-b8e892e3e8bb.json')
+        return key_path
+
+    def _get_spreadsheet_key(self) -> str:
+        spreadsheet_key = '1LVCY6H2iaS05DDwON4YyZ2aWYnFCBYbnZqtpMdn7e6M'
+        return spreadsheet_key
+
+    def get_public_url(self) -> Optional[str]:
+        return (
+            'https://docs.google.com/spreadsheets/d/e/2PACX'
+            '-1vSnyvgqXDq3XPzGz3eLz_8JPwceou10HiEShI0wJ2A8vlosRZc1QhKZ10aOmmQFitv2yPAyBERD2wwx/pubhtml '
+        )