Added check and check_capabilities methods to TestSuites

sciunit/converters.py

@@ -9,7 +9,7 @@
 
 class Converter(object):
     """
-    Base converter class.  
+    Base converter class.
     Only derived classes should be used in applications.
     """
 
@@ -26,7 +26,7 @@ def description(self):
     def _convert(self, score):
         """
         Takes the score attribute of a score instance
-        and recasts it as instance of another score type.  
+        and recasts it as instance of another score type.
         """
         raise NotImplementedError(("The '_convert' method for %s "
                                    "it not implemented." %
@@ -44,7 +44,7 @@ def convert(self, score):
 class NoConversion(Converter):
     """
     Applies no conversion.
-    """    
+    """
 
     def _convert(self, score):
         return score
@@ -60,15 +60,15 @@ def __init__(self, f):
 
     def _convert(self, score):
         return score.__class__(self.f(score))
-        
+
 
 class AtMostToBoolean(Converter):
     """
     Converts a score to pass if its value is at most $cutoff, otherwise False.
     """
     def __init__(self, cutoff):
         self.cutoff = cutoff
-    
+
     def _convert(self, score):
         return BooleanScore(bool(score <= self.cutoff))
 
@@ -79,7 +79,7 @@ class AtLeastToBoolean(Converter):
     """
     def __init__(self, cutoff):
         self.cutoff = cutoff
-    
+
     def _convert(self, score):
         return BooleanScore(score >= self.cutoff)
 

sciunit/scores/collections.py

@@ -1,7 +1,7 @@
 """
-SciUnit score collections, such as arrays and matrices.  
+SciUnit score collections, such as arrays and matrices.
 These collections allow scores to be organized and visualized
-by model, test, or both.  
+by model, test, or both.
 """
 
 from datetime import datetime
@@ -21,8 +21,8 @@ class ScoreArray(pd.Series,SciUnit,TestWeighted):
     """
     Represents an array of scores derived from a test suite.
     Extends the pandas Series such that items are either
-    models subject to a test or tests taken by a model.  
-    Also displays and compute score summaries in sciunit-specific ways.  
+    models subject to a test or tests taken by a model.
+    Also displays and compute score summaries in sciunit-specific ways.
 
     Can use like this, assuming n tests and m models:
 
@@ -74,20 +74,20 @@ def __getattr__(self, name):
         else:
             attr = super(ScoreArray,self).__getattribute__(name)
         return attr
-   
-    @property   
+
+    @property
     def sort_keys(self):
         return self.map(lambda x: x.sort_key)
 
     def mean(self):
-        """Computes a total score for each model over all the tests, 
+        """Computes a total score for each model over all the tests,
         using the sort_key, since otherwise direct comparison across different
         kinds of scores would not be possible."""
 
         return np.dot(np.array(self.sort_keys),self.weights)
-        
+
     def stature(self, test_or_model):
-        """Computes the relative rank of a model on a test compared to other models 
+        """Computes the relative rank of a model on a test compared to other models
         that were asked to take the test."""
 
         return self.sort_keys.rank(ascending=False)[test_or_model]
@@ -100,8 +100,8 @@ class ScoreMatrix(pd.DataFrame,SciUnit,TestWeighted):
     """
     Represents a matrix of scores derived from a test suite.
     Extends the pandas DataFrame such that tests are columns and models
-    are the index.  
-    Also displays and compute score summaries in sciunit-specific ways.  
+    are the index.
+    Also displays and compute score summaries in sciunit-specific ways.
 
     Can use like this, assuming n tests and m models:
 
@@ -113,25 +113,25 @@ class ScoreMatrix(pd.DataFrame,SciUnit,TestWeighted):
     (score_1, ..., score_n)
     """
 
-    def __init__(self, tests, models, 
+    def __init__(self, tests, models,
                  scores=None, weights=None, transpose=False):
         tests, models, scores = self.check_tests_models_scores(\
                                                     tests, models, scores)
         if transpose:
-            super(ScoreMatrix,self).__init__(data=scores.T, index=tests, 
+            super(ScoreMatrix,self).__init__(data=scores.T, index=tests,
                                              columns=models)
         else:
-            super(ScoreMatrix,self).__init__(data=scores, index=models, 
+            super(ScoreMatrix,self).__init__(data=scores, index=models,
                                              columns=tests)
         with warnings.catch_warnings():
-            warnings.filterwarnings("ignore", 
+            warnings.filterwarnings("ignore",
                                     message=(".*Pandas doesn't allow columns "
                                              "to be created via a new "))
             self.tests = tests
             self.models = models
             self.weights_ = [] if not weights else list(weights)
             self.transposed = transpose
-        
+
     show_mean = False
     sortable = False
     direct_attrs = ['score','sort_keys','related_data']
@@ -157,12 +157,12 @@ def __getitem__(self, item):
         raise TypeError("Expected test; model; test,model; or model,test")
 
     def get_test(self, test):
-        return ScoreArray(self.models, 
+        return ScoreArray(self.models,
                           scores=super(ScoreMatrix,self).__getitem__(test),
                           weights=self.weights)
 
     def get_model(self, model):
-        return ScoreArray(self.tests, 
+        return ScoreArray(self.tests,
                           scores=self.loc[model,:],
                           weights=self.weights)
 
@@ -186,20 +186,20 @@ def get_by_name(self, name):
             if test.name == name:
                 return self.__getitem__(test)
         raise KeyError("No model or test with name '%s'" % name)
-  
+
     def __getattr__(self, name):
         if name in self.direct_attrs:
             attr = self.applymap(lambda x: getattr(x,name))
         else:
             attr = super(ScoreMatrix,self).__getattribute__(name)
         return attr
 
-    @property   
+    @property
     def sort_keys(self):
         return self.applymap(lambda x: x.sort_key)
-       
+
     def stature(self, test, model):
-        """Computes the relative rank of a model on a test compared to other models 
+        """Computes the relative rank of a model on a test compared to other models
         that were asked to take the test."""
 
         return self[test].stature(model)
@@ -209,7 +209,7 @@ def T(self):
         return ScoreMatrix(self.tests, self.models, scores=self.values,\
                            weights=self.weights, transpose=True)
 
-    def to_html(self, show_mean=None, sortable=None, colorize=True, *args, 
+    def to_html(self, show_mean=None, sortable=None, colorize=True, *args,
                       **kwargs):
         if show_mean is None:
             show_mean = self.show_mean
@@ -224,10 +224,10 @@ def to_html(self, show_mean=None, sortable=None, colorize=True, *args,
         if sortable:
             self.dynamify(table_id)
         return html
-    
+
     def annotate(self, df, html, show_mean, colorize):
         soup = bs4.BeautifulSoup(html,"lxml")
-        if colorize: 
+        if colorize:
             self.annotate_headers(soup, df, show_mean)
             self.annotate_body(soup, df, show_mean)
         table = soup.find('table')
@@ -286,12 +286,12 @@ def dynamify(self, table_id):
             lib=["%s/jquery.dataTables.min.js" % prefix],
             css=["%s/css/jquery.dataTables.css" % prefix])
         display(js)
-    
+
 
 class ScorePanel(pd.Panel,SciUnit):
     def __getitem__(self, item):
         df = super(ScorePanel,self).__getitem__(item)
         assert isinstance(df,pd.DataFrame), \
             "Only Score Matrices can be accessed by attribute from Score Panels"
         score_matrix = ScoreMatrix(models=df.index, tests=df.columns, scores=df)
-        return score_matrix 
+        return score_matrix

sciunit/suites.py

@@ -1,5 +1,5 @@
 """
-Base class for SciUnit test suites.  
+Base class for SciUnit test suites.
 """
 
 import random
@@ -14,7 +14,7 @@
 class TestSuite(SciUnit,TestWeighted):
     """A collection of tests."""
 
-    def __init__(self, tests, name=None, weights=None, include_models=None, 
+    def __init__(self, tests, name=None, weights=None, include_models=None,
                  skip_models=None, hooks=None):
         self.name = name if name else "Suite_%d" % random.randint(0,1e12)
         self.tests = self.check_tests(tests)
@@ -34,11 +34,11 @@ def __init__(self, tests, name=None, weights=None, include_models=None,
     """The sequence of tests that this suite contains."""
 
     include_models = []
-    """List of names or instances of models to judge 
+    """List of names or instances of models to judge
     (all passed to judge are judged by default)."""
 
     skip_models = []
-    """List of names or instances of models to not judge 
+    """List of names or instances of models to not judge
     (all passed to judge are judged by default)."""
 
     def check_tests(self, tests):
@@ -74,12 +74,35 @@ def check_models(self, models):
                                  "a model or iterable."))
         return models
 
-    def judge(self, models, 
+    def check(self, models, skip_incapable=True, stop_on_error=True):
+        """Like judge, but without actually running the test.
+
+        Just returns a ScoreMatrix indicating whether each model can take
+        each test or not.  A TBDScore indicates that it can, and an NAScore
+        indicates that it cannot.
+        """
+        sm = ScoreMatrix(self.tests, models)
+        print(sm.shape)
+        for test in self.tests:
+            for model in models:
+                sm.loc[model, test] = test.check(model)
+        return sm
+
+    def check_capabilities(self, model, skip_incapable=False):
+        """Check model capabilities against those required by the suite.
+
+        Returns a list of booleans (one for each test in the suite)
+        corresponding to whether the test's required capabilities are satisfied
+        by the model.
+        """
+        return [test.check_capabilities(model,
+                skip_incapable=skip_incapable) for test in self.tests]
+
+    def judge(self, models,
               skip_incapable=False, stop_on_error=True, deep_error=False):
         """Judges the provided models against each test in the test suite.
            Returns a ScoreMatrix.
         """
-
         models = self.check_models(models)
         sm = ScoreMatrix(self.tests, models, weights=self.weights)
         for model in models:
@@ -96,21 +119,21 @@ def is_skipped(self, model):
         skip = self.include_models and \
                not any([model.is_match(x) for x in self.include_models])
         # Skip if model found in skip_models
-        if not skip: 
+        if not skip:
             skip = any([model.is_match(x) for x in self.skip_models])
-        return skip 
-        
-    def judge_one(self, model, test, sm, 
+        return skip
+
+    def judge_one(self, model, test, sm,
                   skip_incapable=True, stop_on_error=True, deep_error=False):
         """Judge model and put score in the ScoreMatrix"""
-        
+
         if self.is_skipped(model):
             score = NoneScore(None)
-        else:    
-            log('Executing test <i>%s</i> on model <i>%s</i>' % (test,model), 
+        else:
+            log('Executing test <i>%s</i> on model <i>%s</i>' % (test,model),
             end="... ")
-            score = test.judge(model, skip_incapable=skip_incapable, 
-                                  stop_on_error=stop_on_error, 
+            score = test.judge(model, skip_incapable=skip_incapable,
+                                  stop_on_error=stop_on_error,
                                   deep_error=deep_error)
             log('Score is <a style="color: rgb(%d,%d,%d)">' % score.color()
             + '%s</a>' % score)
@@ -138,8 +161,8 @@ def set_verbose(self, verbose):
     def from_observations(cls, tests_info, name=None):
         """Instantiate a test suite with name 'name' and information about tests
         in 'tests_info', as [(TestClass1,obs1),(TestClass2,obs2),...].
-        The desired test name may appear as an optional third item in the 
-        tuple, e.g. (TestClass1,obse1,"my_test").  The same test class may be 
+        The desired test name may appear as an optional third item in the
+        tuple, e.g. (TestClass1,obse1,"my_test").  The same test class may be
         used multiple times, e.g. [(TestClass1,obs1a),(TestClass1,obs1b),...].
         """
 
@@ -154,4 +177,4 @@ def from_observations(cls, tests_info, name=None):
         return cls(tests, name=name)
 
     def __str__(self):
-        return '%s' % self.name
+        return '%s' % self.name