Add text comparison metrics to Ben's PR

.pre-commit-config.yaml

@@ -51,4 +51,9 @@ repos:
             "GeoAlchemy2",
             "psycopg2-binary",
             "pgvector",
+            "datasets >= 2.20.0",
+            "absl-py",
+            "nltk",
+            "rouge_score",
+            "evaluate",
           ]

api/Dockerfile

@@ -1,18 +1,17 @@
-FROM python:3.10-alpine
+FROM python:3.10-slim
 
-ARG VERSION="0.0.0-dev"
+ARG VALOR_VERSION="0.0.0-dev"
 
-RUN apk add --update --no-cache build-base libpq-dev gcc libffi-dev
+RUN apt-get update && apt-get install build-essential libpq-dev -y
 
 COPY ./pyproject.toml /src/
 
 WORKDIR /src
 RUN python -m pip install -U pip
 # when building, we'll use setuptools_scm on the hostmachine to get the version
-# and then pass it to the build arg VERSION. this is so we don't need to install
+# and then pass it to the build arg VALOR_VERSION. this is so we don't need to install
 # git and put .git (which setuptools_scm needs to determine the version) in the container
-RUN SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION} python -m pip install .
+RUN SETUPTOOLS_SCM_PRETEND_VERSION=${VALOR_VERSION} python -m pip install .
 COPY ./valor_api /src/valor_api
-RUN apk del build-base
-USER 65532:65532
+
 CMD ["uvicorn", "valor_api.main:app", "--host", "0.0.0.0", "--log-level", "warning"]

api/pyproject.toml

@@ -19,6 +19,11 @@ dependencies = [
     "pgvector",
     "openai",
     "mistralai",
+    "datasets >= 2.20.0",
+    "absl-py",
+    "nltk",
+    "rouge_score",
+    "evaluate"
 ]
 
 [build-system]