use aggregators as a uniform incarnation for the across-instances computation for both instance and global metrics

[dafnapension]

Thus simplified and thereby extended grouping and filtering over to global metrics and to bulk-instance metrics

[elronbandel]

Suggested change

	class Aggregator(Artifact):
	@abstractmethod
	def aggregate_one_group_score_named(
	self, instances: List[Dict[str, Any]], score_names: List[str]
	) -> dict:
	pass
	class Aggregator(Artifact):
	"""
	Aggregate list of instances to a dictionary of scores.
	""""
	score_names: List[str]
	@abstractmethod
	def aggregate(
	self, instances: List[Dict[str, Any]]
	) -> Dict[str, Any]:
	pass
	def __call__(self, instances):
	return self.aggregate(instances)

[elronbandel]

Suggested change

	class SimpleAggregator(Aggregator):
	aggregating_func: Callable[[List[Dict[str, Any]], str], float]
	def aggregate_one_group_score_named(
	self, instances: List[Dict[str, Any]], score_names: List[str]
	) -> dict:
	result = {}
	for score_name in score_names:
	result[score_name] = self.aggregating_func(instances, score_name)
	return result
	def average_item_scores(instances: List[dict], score_name: str) -> float:
	"""Calculate mean of a set of instance scores (given by score_name), omitting NaN values.
	Args:
	instances: list of dicts of each instance's instance scores.
	score_name: score field names to compute the mean for.
	"""
	return nan_mean(
	[instance["score"]["instance"][score_name] for instance in instances]
	)
	def max_item_scores(instances: List[dict], score_name: str) -> float:
	"""Calculate max of a set of instance scores (given by score_name), omitting NaN values.
	Args:
	instances: list of dicts of each instance's instance scores.
	score_name: score field names to compute the mean for.
	"""
	return nan_max(
	[instance["score"]["instance"][score_name] for instance in instances]
	)
	class AverageItemsAggregator(SimpleAggregator):
	aggregating_func = Field(default_factory=lambda: average_item_scores)
	class Mean(Aggregator):
	def aggregate(
	self, instances: List[Dict[str, Any]]
	) -> dict:
	result = {}
	for score_name in self.score_names:
	result[score_name] = nan_mean(
	[instance["score"]["instance"][score_name] for instance in instances]
	)
	class Max(Aggregator):
	def aggregate(
	self, instances: List[Dict[str, Any]]
	) -> dict:
	result = {}
	for score_name in self.score_names:
	result[score_name] = nan_max(
	[instance["score"]["instance"][score_name] for instance in instances]
	)
	class ConfidenceInterval(Aggregator):
	aggregator: Aggregator = Mean()
	def aggregate(
	self, instances: List[Dict[str, Any]]
	) -> dict:
	results = []
	for sample in sample(instances):
	result = self.aggregate(sample)
	results.append(result)
	result["ci_low"], results["ci_high"] = ci(results)
	class Filter(Aggregator):
	aggregator: Aggregator = Mean()
	filter: Func
	def aggregate(
	self, instances: List[Dict[str, Any]]
	) -> dict:
	instances = [instance for instance in instances if self.filter(instance)]
	return self.aggregate(instances)
	class Group(Aggregator):
	group_aggregator: Aggregator = Mean()
	all_groups_aggregator: Aggregator = Mean()
	results = {}
	group_results =[]
	for group, group_name in split_to_groups(instances):
	result = self.group_aggregator(group)
	group_results.append(result)
	update_result(result, group_name)
	results.update(self. all_groups_aggregator(group_results))
	return results

[elronbandel]

class Mapper:
    def map(instance: Dict[str, Any]): -> Dict[str, Any]
         pass
     
     def __call__(instances):
            return [self.map(instance) for instance in instances]
        
class Metric:
     aggregate: Aggregator = Group(ConfigenceInterval(Mean(fields=["f1"]), group_by="group_id")
     # aggregator: Aggregator = Group(ConfigenceInterval(RougeAggregator()), group_by="group_id")
     # map: Mapper = RougeInstanceScore()
     
     def compute(instances):
         instances = self.map(instances)
         return self.aggregate(instances)
     

So every metric has one and only aggregator

[elronbandel]

Suggested change

	aggregator: Aggregator = Field(default_factory=lambda: AverageItemsAggregator())
	aggregator: Aggregator = Mean()

[elronbandel]

Suggested change

	aggregation_func = AverageItemsAggregator().aggregate_one_group_score_named
	aggregation_func = self.aggregator

[elronbandel]

See comments

[dafnapension]

Maybe return to this after the war ends

[dafnapension]

too complicated to maintain rebase-able. So closing for now.