Further renaming (#47)

Extra work to disambiguate from a testing framework for the Ada language.

.github/workflows/python-app.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-12, windows-latest]
         python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:

.github/workflows/python-wheel-build.yaml

@@ -49,7 +49,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os: [macos-12-latest, ubuntu-latest, windows-latest]
         python-version: ['3.8', '3.9', '3.10']
 
     steps:
@@ -61,7 +61,7 @@ jobs:
     - name: Remove AdaTest source directory
       uses: JesseTG/rm@v1.0.3
       with:
-        path: adatest
+        path: adaptivetesting
     - name: Download wheel artifact
       uses: actions/download-artifact@v3
       with:

.gitignore

@@ -7,7 +7,7 @@ node_modules
 .jekyll-cache
 .ipynb_checkpoints
 .sass-cache
-adatest.egg-info
+adaptivetesting.egg-info
 dist/gadfly-*
 local_scratch*
 wandb

README.md

@@ -18,23 +18,23 @@ adaptive-testing can test any NLP model you can call with a python function, her
 
 ```python
 import transformers
-import adatest
+import adaptivetesting
 
 # create a HuggingFace sentiment analysis model
 classifier = transformers.pipeline("sentiment-analysis", return_all_scores=True)
 
 # specify the backend generator used to help you write tests
-generator = adatest.generators.OpenAI('curie', api_key=OPENAI_API_KEY)
+generator = adaptivetesting.generators.OpenAI('curie', api_key=OPENAI_API_KEY)
 
 # ...or you can use an open source generator
 #neo = transformers.pipeline('text-generation', model="EleutherAI/gpt-neo-125M")
-#generator = adatest.generators.Transformers(neo.model, neo.tokenizer)
+#generator = adaptivetesting.generators.Transformers(neo.model, neo.tokenizer)
 
 # create a new test tree
-tests = adatest.TestTree("hotel_reviews.csv")
+tests = adaptivetesting.TestTree("hotel_reviews.csv")
 
 # adapt the tests to our model to launch a notebook-based testing interface
-# (wrap with adatest.serve to launch a standalone server)
+# (wrap with adaptivetesting.serve to launch a standalone server)
 tests.adapt(classifier, generator, auto_save=True)
 ```
 

adatest/_model.py → adaptivetesting/_model.py

@@ -3,7 +3,7 @@
 
 
 class Model():
-    """ This wraps models used in AdaTest so that have a consistent interface.
+    """ This wraps models used in Adaptive Testing so that have a consistent interface.
 
     This should eventually just be the Model class from SHAP, but we keep a simple version here for now
     so we can easily update it during initial development.
@@ -12,7 +12,7 @@ class Model():
     def __new__(cls, model, *args, **kwargs):
         """ If we are wrapping a model that is already a Model, we just return it.
         """
-        if shap.utils.safe_isinstance(model, "adatest.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
+        if shap.utils.safe_isinstance(model, "adaptivetesting.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
             return model
         else:
             return super().__new__(cls)
@@ -32,7 +32,7 @@ def __init__(self, model, output_names=None, **kwargs):
         """
 
         # finish early if we are wrapping an object that is already a Model
-        if shap.utils.safe_isinstance(model, "adatest.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
+        if shap.utils.safe_isinstance(model, "adaptivetesting.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
             if output_names is not None:
                 self.output_names = output_names
             assert len(kwargs) == 0

adatest/_prompt_builder.py → adaptivetesting/_prompt_builder.py

@@ -2,7 +2,7 @@
 import logging
 import re
 import urllib.parse
-import adatest
+import adaptivetesting
 from .embedders import cos_sim
 from .utils import is_subtopic
 log = logging.getLogger(__name__)
@@ -53,7 +53,7 @@ def __call__(self, test_tree, topic, score_column, repetitions=1, filter="", sug
 
         Parameters
         ----------
-        test_tree : adatest.TestTree
+        test_tree : adaptivetesting.TestTree
             The test tree to generate prompts from.
         
         topic : str
@@ -158,13 +158,13 @@ def __call__(self, test_tree, topic, score_column, repetitions=1, filter="", sug
             if self.prompt_diversity:
                 sim_avoidance = np.zeros(len(ids))
                 if suggest_topics:
-                    embeddings_arr = np.vstack(adatest.embed(
+                    embeddings_arr = np.vstack(adaptivetesting.embed(
                         [urllib.parse.unquote(test_tree.loc[id, "topic"].split("/")[-1]) for id in ids]
                     ))
                 else:
                     embeddings_arr = np.hstack([
-                        np.vstack(adatest.embed([test_tree.loc[id, "input"] for id in ids])),
-                        np.vstack(adatest.embed([test_tree.loc[id, "output"] for id in ids]))
+                        np.vstack(adaptivetesting.embed([test_tree.loc[id, "input"] for id in ids])),
+                        np.vstack(adaptivetesting.embed([test_tree.loc[id, "output"] for id in ids]))
                     ])
                 similarities = cos_sim(embeddings_arr, embeddings_arr)
             hard_avoidance = np.zeros(len(ids))

adatest/_scorer.py → adaptivetesting/_scorer.py

@@ -5,15 +5,15 @@
 import itertools
 import shap
 from ._model import Model
-import adatest
+import adaptivetesting
 
 log = logging.getLogger(__name__)
 
 class Scorer():
     def __new__(cls, model, *args, **kwargs):
         """ If we are wrapping an object that is already a Scorer, we just return it.
         """
-        if shap.utils.safe_isinstance(model, "adatest.Scorer"):
+        if shap.utils.safe_isinstance(model, "adaptivetesting.Scorer"):
             return model
         else:
             return super().__new__(cls)
@@ -23,9 +23,9 @@ def __init__(self, model):
         """
 
         # ensure we have a model of type Model
-        if shap.utils.safe_isinstance(getattr(self, "model", None), "adatest.Model") or shap.utils.safe_isinstance(getattr(self, "model", None), "shap.models.Model"):
+        if shap.utils.safe_isinstance(getattr(self, "model", None), "adaptivetesting.Model") or shap.utils.safe_isinstance(getattr(self, "model", None), "shap.models.Model"):
             pass
-        elif shap.utils.safe_isinstance(model, "adatest.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
+        elif shap.utils.safe_isinstance(model, "adaptivetesting.Model") or shap.utils.safe_isinstance(model, "shap.models.Model"):
             self.model = model
         else:
             self.model = Model(model)
@@ -34,7 +34,7 @@ def __init__(self, model):
         if self.__class__ is Scorer:
 
             # finish early if we are wrapping an object that is already a Scorer (__new__ will have already done the work)
-            if shap.utils.safe_isinstance(model, "adatest.Scorer"):
+            if shap.utils.safe_isinstance(model, "adaptivetesting.Scorer"):
                 return
 
             # see if we are scoring a generator or a classifier
@@ -143,7 +143,7 @@ def __call__(self, tests, eval_ids):
             out_probs[i] = np.column_stack(out_probs[i]) # the probability of a set of items is the prob of the min item
 
         # compute the embeddings as a batch (this fills a cache we will use when scoring below)
-        adatest.embed(list(tests.loc[eval_ids, "input"]))
+        adaptivetesting.embed(list(tests.loc[eval_ids, "input"]))
 
         # score all the tests
         scores = []

adatest/_server.py → adaptivetesting/_server.py

@@ -88,7 +88,7 @@ async def topic_handler(request):
             interface_html = f"""
 <html>
   <head>
-    <title>AdaTest</title>
+    <title>Adaptive Testing</title>
   </head>
   <body style="font-family: Helvetica Neue, Helvetica, Arial, sans-serif; margin-right: 20px; font-size: 14px;">
     {test_tree_browser._repr_html_(prefix=prefix, environment="web", websocket_server=prefix+"/_ws")}
@@ -138,7 +138,7 @@ async def login_handler(request):
         return web.Response(text=f"""
 <html>
   <head>
-    <title>AdaTest Login</title>
+    <title>Adaptive Testing Login</title>
   </head>
   <body style="font-family: arial">
     <form method="post" action="/_auth" style="margin-top: 80px">
@@ -327,7 +327,7 @@ async def make_app():
             ])
 
         policy = SessionIdentityPolicy()
-        setup_security(app, policy, AdaTestPolicy())
+        setup_security(app, policy, AdaptiveTestingPolicy())
 
         return app
 
@@ -364,7 +364,7 @@ async def stop_server(state):
     finally:
         loop.run_until_complete(stop_server(state))
 
-class AdaTestPolicy(AbstractAuthorizationPolicy):
+class AdaptiveTestingPolicy(AbstractAuthorizationPolicy):
     async def authorized_userid(self, identity):
         """Retrieve authorized user id.
         Return the user_id of the user identified by the identity

adatest/_test_tree.py → adaptivetesting/_test_tree.py

@@ -10,7 +10,7 @@
 from ._test_tree_browser import TestTreeBrowser, is_subtopic
 from ._model import Model
 from ._topic_model import TopicLabelingModel, TopicMembershipModel
-import adatest
+import adaptivetesting
 from pathlib import Path
 
 # class TestTreeIterator():
@@ -48,7 +48,7 @@ def __init__(self, tests=None, labeling_model=TopicLabelingModel, membership_mod
             Assigns an index to underlying tests frame, or auto generates if not provided.
 
         compute_embeddings: boolean
-            If True, use the global adatest.embed to build embeddings of tests in the TestTree.
+            If True, use the global adaptivetesting.embed to build embeddings of tests in the TestTree.
 
         kwargs : dict
             Additional keyword arguments are passed to the pandas DataFrame constructor.
@@ -289,7 +289,7 @@ def topic_has_subtopics(self, target_topic: str) -> bool:
         )
         return has_subtopics_df.any()
 
-    def adapt(self, scorer=None, generator=adatest.generators.OpenAI(), auto_save=False, user="anonymous", recompute_scores=False, drop_inactive_score_columns=False,
+    def adapt(self, scorer=None, generator=adaptivetesting.generators.OpenAI(), auto_save=False, user="anonymous", recompute_scores=False, drop_inactive_score_columns=False,
               max_suggestions=100, suggestion_thread_budget=0.5, prompt_builder=PromptBuilder(), active_generator="default", starting_path="",
               score_filter=-1e10, topic_model_scale=0): # TODO: remove active_generator and replace with the ability to set the generator?
         """ Apply this test tree to a scorer/model and browse/edit the tests to adapt them to the target model.
@@ -299,12 +299,12 @@ def adapt(self, scorer=None, generator=adatest.generators.OpenAI(), auto_save=Fa
         
         Parameters
         ----------
-        scorer : adatest.Scorer or callable
+        scorer : adaptivetesting.Scorer or callable
             The scorer (that wraps a model) to used to score the tests. If a function is provided, it will be wrapped in a scorer.
             Passing a dictionary of scorers will score multiple models at the same time. Note that the models are expected to take
             a list of strings as input, and output either a classification probability vector or a string.
 
-        generator : adatest.Generator or dict[adatest.Generators]
+        generator : adaptivetesting.Generator or dict[adaptivetesting.Generators]
             A source to generate new tests from. Currently supported generator types are language models, existing test trees, or datasets.
 
         auto_save : bool
@@ -329,12 +329,12 @@ def adapt(self, scorer=None, generator=adatest.generators.OpenAI(), auto_save=Fa
             different randomized LM prompt for test generation, so more threads will result in more diversity, but come at the cost
             of reading more prompt variations.
 
-        prompt_builder : adatest.PromptBuilder
+        prompt_builder : adaptivetesting.PromptBuilder
             A prompt builder to use when generating prompts for new tests. This object controls how the LM prompts
             are created when generating new tests.
 
         active_generator : "default", or a key name if generators is a dictionary
-            Which generator from adatest.generators to use when generating new tests. This should always be set to "default" if
+            Which generator from adaptivetesting.generators to use when generating new tests. This should always be set to "default" if
             generators is just a single generator and not a dictionary of generators.
 
         starting_path : str
@@ -417,7 +417,7 @@ def _cache_embeddings(self, ids=None):
         # all_strings.append("__suggestions__")
 
         # we don't use the output of the embedding, just do this to get the embeddings cached
-        adatest.embed(all_strings)
+        adaptivetesting.embed(all_strings)
 
     def impute_labels(self):
         """ Impute missing labels in the test tree. """
@@ -458,7 +458,7 @@ def impute_labels(self):
     #         to_embed.append(input)
     #         to_embed.append(output)
     #         topics[topic].append((i, len(to_embed) - 2, len(to_embed) - 1))
-    #     embeddings = adatest.embed(to_embed)
+    #     embeddings = adaptivetesting.embed(to_embed)
     #     features = [None for i in range(len(topical_io_pairs))]
     #     for topic in topics:
     #         features = []

adatest/_test_tree_browser.py → adaptivetesting/_test_tree_browser.py

@@ -5,7 +5,7 @@
 import re
 from tqdm import tqdm
 
-from adatest.generators import TestTreeSource
+from adaptivetesting.generators import TestTreeSource
 
 from .comm import JupyterComm
 import uuid
@@ -16,7 +16,7 @@
 import statistics
 from threading import Timer
 from ._scorer import expand_template, clean_template, Scorer
-import adatest # Need to import like this to prevent circular dependencies
+import adaptivetesting # Need to import like this to prevent circular dependencies
 import urllib.parse
 from .utils import is_subtopic
 
@@ -115,12 +115,12 @@ def __init__(self, test_tree, scorer, generators, user, auto_save, recompute_sco
         if not isinstance(self.generators, dict):
             self.generators = {'generator': self.generators}
 
-        if adatest.default_generators is not None: # Merge default generators into generators
-            self.generators = {**self.generators, **adatest.default_generators}
+        if adaptivetesting.default_generators is not None: # Merge default generators into generators
+            self.generators = {**self.generators, **adaptivetesting.default_generators}
 
         # Find and cast any TestTrees in generators to TestTreeSource
         for generator_name, generator in self.generators.items():
-            if isinstance(generator, adatest._test_tree.TestTree): # TODO: make this autoreload friendly
+            if isinstance(generator, adaptivetesting._test_tree.TestTree): # TODO: make this autoreload friendly
                 self.generators[generator_name] = TestTreeSource(generator) 
 
         # get a reference to the active backend object
@@ -301,7 +301,7 @@ def interface_event(self, msg):
             self.test_tree.retrain_topic_membership_model(self.current_topic)
             self._generate_suggestions(filter=msg.get("filter", ""))
             # if self._active_generator_obj is None:
-            #     self._suggestions_error = "No AdaTest generator has been set!"
+            #     self._suggestions_error = "No adaptivetesting generator has been set!"
             # else:
             #     self._generate_suggestions(filter=msg[k].get("filter", ""))
             # # try: