Custom Comparator for eval-agent

This repo contains the framework with examples for running custom comparison of solutions evaluated by eval-agent.

This README file is a brief overview, but authors are encouraged to examine all the files in the repository.

For any problem with a custom checker, authors only need to implement and test comparator.cc:

#include "comparator.h"

namespace evalagent {

bool Comparator::Compare(
    std::ifstream* input,  // A reading file stream with the input data.
    std::ifstream* expected,  // A reading file stream with the expected output.
    std::ifstream* actual  // A reading file stream with the solution output.
    ) {
  // TODO: Implement comparison.
}

}  // namespace evalagent

To simplify common operations, the following helper functions are available:

// Returns an optional token of TokenType read from the specified file.
// Examples:
//   const Optional<int> int_token = ReadToken<int>(file);
//   const Optional<double> double_token = ReadToken<double>(file);
//   const Optional<std::string> str_token = ReadToken<std::string>(file);
template <typename TokenType>
Optional<TokenType> ReadToken(std::ifstream* file);

// Returns an optional string line read from the specified file.
Optional<std::string> ReadLine(std::ifstream* file);

// Returns true if the optionals are equal by using the provided comparator.
template <typename ValueType, class Compare>
bool AreEqual(Optional<ValueType> expected, Optional<ValueType> actual,
              Compare compare);

// Same as AreEqual, but using the default equality operator ==.
template <typename ValueType>
bool AreEqual(Optional<ValueType> expected, Optional<ValueType> actual);

// Returns true if both sequences are equal by using the provided comparator.
// Sequences are also tested for being of equal length.
template <typename ValueType, class Compare>
bool AreSequencesEqual(
    std::ifstream* expected_file,
    std::ifstream* actual_file,
    Compare compare);

// Same as AreSequencesEqual, but using the default equality operator ==.
template <typename ValueType>
bool AreSequencesEqual(
  std::ifstream* expected_file, std::ifstream* actual_file);

// Returns true if both string line sequences are equal by using the provided
// comparator. Line sequences are also tested for being of equal length.
template <class Compare>
bool AreLinesEqual(
    std::ifstream* expected_file,
    std::ifstream* actual_file,
    Compare compare);

// Same as AreLinesEqual, but using the default equality operator ==.
bool AreLinesEqual(
    std::ifstream* expected_file,
    std::ifstream* actual_file);

// Returns a comparator which evaluates expected == actual.
template <typename ValueType>
std::function<bool(ValueType, ValueType)> Exactly();

// Returns a comparator which evaluates |expected - actual| < epsilon.
// All inputs are of the same decimal type, e.g. float or double.
// Examples:
//   AreEqual(0.5, 0.4, WithPrecision(0.1))  // false
//   AreEqual(0.5, 0.401, WithPrecision(0.1))  // true
template <typename DecimalType>
std::function<bool(DecimalType, DecimalType)> WithPrecision(
    DecimalType epsilon);

Example 1: Most common (default) comparison - line by line of text:

#include "comparator.h"

namespace evalagent {

bool Comparator::Compare(
    std::ifstream* input,
    std::ifstream* expected,
    std::ifstream* actual) {
  return AreLinesEqual(expected, actual);
}

}  // namespace evalagent

Example 2: Compare two doubles with a 6 digit precision:

#include "comparator.h"

namespace evalagent {

bool Comparator::Compare(
    std::ifstream* input,
    std::ifstream* expected,
    std::ifstream* actual) {
  return AreEqual<double>(
      ReadToken<double>(expected),
      ReadToken<double>(actual),
      WithPrecision(1e-6));
}

}  // namespace evalagent

Example 3: Check if two sequences of doubles match:

#include "comparator.h"

namespace evalagent {

bool Comparator::Compare(
    std::ifstream* input,
    std::ifstream* expected,
    std::ifstream* actual) {
  return AreSequencesEqual<double>(expected, actual, WithPrecision(1e-6));
}

}  // namespace evalagent

Example 4: Check if two sentences match by content using a custom matcher:

#include "comparator.h"

#include <algorithm>

namespace evalagent {

// Returns the lowercase version of the original string.
std::string lowercase(const std::string& original) {
  std::string r = original;
  std::transform(
      r.begin(), r.end(), r.begin(),
      [](unsigned char c){ return std::tolower(c); });
  return r;
}

// Returns a comparison function to evaluate content ignoring the letter case.
std::function<bool(std::string, std::string)> ByContent() {
  return [](std::string a, std::string b) {
    return lowercase(a) == lowercase(b);
  };
}

bool Comparator::Compare(
    std::ifstream* input,
    std::ifstream* expected,
    std::ifstream* actual) {
  return AreEqual(
      ReadLine(expected),
      ReadLine(actual),
      ByContent());
}

}  // namespace evalagent

Testing examples in this project

# All examples
./test.sh

# Single example
./test.sh sequences
./test.sh strings