spec.yaml

openapi: "3.0.0"

info:
  title: Gentrace API
  description: These API routes are designed to ingest events from clients.
  version: ${npm_package_version}

tags:
  - name: v1
    description: Endpoints for core API functionality (version 1)
  - name: v2
    description: Endpoints for core API functionality (version 2)
  - name: Feedback
    description: Endpoints for capturing user feedback for runs

servers:
  - url: https://gentrace.ai/api

security:
  - bearerAuth: []

paths:
  /v1/run:
    post:
      tags:
        - v1
      summary: Create a run
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/RunRequest"
      responses:
        "200":
          description: Stored run
          content:
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/RunResponse"
            application/json:
              schema:
                $ref: "#/components/schemas/RunResponse"
        "400":
          description: Bad request
        "500":
          description: Internal server error

  /v1/feedback:
    post:
      tags:
        - v1
      summary: Submit feedback
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/FeedbackRequest"
      responses:
        "200":
          description: Feedback submitted successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/FeedbackResponse"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/FeedbackResponse"
        "400":
          description: Invalid feedback payload
        "500":
          description: Server error

  /v1/test-case:
    get:
      tags:
        - v1
      summary: Get test cases for a pipeline
      parameters:
        - in: query
          name: pipelineId
          schema:
            type: string
            format: uuid
          description: The ID of the Pipeline to retrieve test cases for
        - in: query
          name: pipelineSlug
          schema:
            type: string
          description: The slug of the Pipeline to retrieve test cases for
      responses:
        "200":
          description: Test cases retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  testCases:
                    type: array
                    items:
                      $ref: "#/components/schemas/TestCase"
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  testCases:
                    type: array
                    items:
                      $ref: "#/components/schemas/TestCase"
        "400":
          description: Invalid pipeline ID
        "500":
          description: Server error

    post:
      tags:
        - v1
      summary: Create a new test case
      requestBody:
        required: true
        content:
          application/json:
            schema:
              oneOf:
                - $ref: "#/components/schemas/CreateSingleTestCase"
                - $ref: "#/components/schemas/CreateMultipleTestCases"
      responses:
        "200":
          description: Test case(s) created successfully
          content:
            application/json:
              schema:
                oneOf:
                  - type: object
                    properties:
                      caseId:
                        type: string
                        description: ID of the created test case
                  - type: object
                    properties:
                      creationCount:
                        type: integer
                        description: Number of created test cases
            application/json; charset=utf-8:
              schema:
                oneOf:
                  - type: object
                    properties:
                      caseId:
                        type: string
                        description: ID of the created test case
                  - type: object
                    properties:
                      creationCount:
                        type: integer
                        description: Number of created test cases
        "400":
          description: Bad request
        "500":
          description: Server error

    patch:
      tags:
        - v1
      summary: Update an existing test case
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/UpdateTestCase"
      responses:
        "200":
          description: Test case updated successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  caseId:
                    type: string
                    description: ID of the updated test case
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  caseId:
                    type: string
                    description: ID of the updated test case

        "400":
          description: Bad request
        "500":
          description: Server error

  /v1/test-result:
    get:
      tags:
        - v1
      summary: Get test results for a pipeline
      parameters:
        - name: pipelineSlug
          in: query
          required: true
          description: The slug of the pipeline
          schema:
            type: string
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: object
                properties:
                  testResults:
                    type: array
                    items:
                      $ref: '#/components/schemas/TestResult'
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  testResults:
                    type: array
                    items:
                      $ref: '#/components/schemas/TestResult'

        '400':
          description: Invalid request parameters
        '404':
          description: Results not found

    post:
      tags:
        - v1
      summary: Create a new test result from runs
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                pipelineSlug:
                  type: string
                  description: The pipeline slug to create the test result for. Only one of pipelineSlug or pipelineId is required.
                pipelineId:
                  type: string
                  description: The pipeline ID to create the test result for. Only one of pipelineSlug or pipelineId is required.
                collectionMethod:
                  type: string
                  description: The method used to collect the test runs
                  enum:
                    - manual
                    - runner
                branch:
                  type: string
                  description: The branch that the test result was created from
                  nullable: true
                commit:
                  type: string
                  description: The commit that the test result was created from
                  nullable: true
                name:
                  type: string
                  description: The name of the test result
                  nullable: true
                metadata:
                  type: object
                  additionalProperties:
                    $ref: '#/components/schemas/MetadataValueObject'
                  nullable: true
                testRuns:
                  type: array
                  items:
                    type: object
                    properties:
                      id:
                        type: string
                        format: uuid
                        description: The ID of the test run
                        nullable: true
                      caseId:
                        type: string
                        format: uuid
                        description: The ID of the test case
                      metadata:
                        type: object
                        additionalProperties:
                          $ref: '#/components/schemas/MetadataValueObject'
                        nullable: true
                      stepRuns:
                        type: array
                        description: Use outputs.steps insteads.
                        items:
                          $ref: '#/components/schemas/StepRun'
                    required:
                      - caseId
                      - stepRuns
              required:
                - testRuns
      responses:
        "200":
          description: Test result created successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  resultId:
                    type: string
                    format: uuid
                required:
                - resultId
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  resultId:
                    type: string
                    format: uuid
                required:
                - resultId

  /v1/test-result/{id}:
    get:
      tags:
        - v1
      summary: Fetch test result by ID
      parameters:
        - name: id
          in: path
          description: The ID of the test result
          required: true
          schema:
            type: string
            format: uuid
      responses:
        200:
          description: Test result retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ExpandedTestResult'
            application/json; charset=utf-8:
              schema:
                $ref: '#/components/schemas/ExpandedTestResult'
        400:
          description: Bad request.
        404:
          description: Test result not found.
        500:
          description: Internal server error.


  /v1/test-result/status:
    get:
      tags:
        - v1
      summary: Get status of the test result
      parameters:
        - in: query
          name: id
          schema:
            type: string
            format: uuid
          required: true
          description: The ID of the test result
      responses:
        "200":
          description: Test result status retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  total:
                    type: number
                  pending:
                    type: number
                  failure:
                    type: number
                  done:
                    type: number
                required:
                  - total
                  - pending
                  - failure
                  - done
                        
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  total:
                    type: number
                  pending:
                    type: number
                  failure:
                    type: number
                  done:
                    type: number
                required:
                  - total
                  - pending
                  - failure
                  - done

  /v1/test-result-simple:
    post:
      tags:
        - v1
      summary: Create a new test result from test runs
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                pipelineSlug:
                  type: string
                  description: Slug for the pipeline
                branch:
                  type: string
                  description: The branch that the test result was created from
                  nullable: true
                commit:
                  type: string
                  description: The commit that the test result was created from
                  nullable: true
                name:
                  type: string
                  description: The name of the test result
                  nullable: true
                metadata:
                  type: object
                  additionalProperties:
                    $ref: '#/components/schemas/MetadataValueObject'
                  nullable: true
                testRuns:
                  type: array
                  items:
                    type: object
                    properties:
                      id:
                        type: string
                        format: uuid
                        description: The ID of the test run
                        nullable: true
                      caseId:
                        type: string
                        format: uuid
                        description: The ID of the test case
                      inputs:
                        type: object
                        additionalProperties: true
                        description: The input data for the test case
                      outputs:
                        type: object
                        additionalProperties: true
                        description: The returned outputs for the test case
                    required:
                      - caseId
                      - inputs
              required:
                - pipelineSlug
                - testRuns
      responses:
        "200":
          description: Test result created successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  resultId:
                    type: string
                    format: uuid
                required:
                - resultId
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  resultId:
                    type: string
                    format: uuid
                required:
                - resultId

  /v1/pipelines:
    get:
      tags:
        - v1
      summary: Get pipelines, optionally filtered by label
      parameters:
        - in: query
          name: label
          schema:
            type: string
          description: The label to filter pipelines by
        - in: query
          name: slug
          schema:
            type: string
          description: The slug to filter pipelines by
      responses:
        "200":
          description: Pipelines retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  pipelines:
                    type: array
                    items:
                      allOf:
                      - $ref: "#/components/schemas/Pipeline"
                      - type: object
                        properties:
                          cases:
                            type: array
                            items:
                              $ref: "#/components/schemas/TestCase"
                required:
                  - pipelines
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  pipelines:
                    type: array
                    items:
                      allOf:
                      - $ref: "#/components/schemas/Pipeline"
                      - type: object
                        properties:
                          cases:
                            type: array
                            items:
                              $ref: "#/components/schemas/TestCase"
                required:
                  - pipelines
        "400":
          description: Invalid pipeline ID
        "500":
          description: Server error

  /v1/files/upload:
   post:
      tags:
        - v1
      summary: Upload an image file
      parameters:
        - in: query
          name: org
          description: (If not using an organization API Key) the ID of the organization
          schema:
            type: string
      requestBody:
        required: true
        content:
          multipart/form-data:
            schema:
              type: object
              properties:
                file:
                  type: string
                  format: binary
                  description: "The file to upload."
      responses:
        '201':
          description: Image uploaded successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  url:
                    type: string
                    description: URL of the uploaded image
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  url:
                    type: string
                    description: URL of the uploaded image
        '400':
          description: Bad request. Reasons can be missing org ID, missing file name or file too large.
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  message:
                    type: string
        '401':
          description: Unauthorized. Invalid API key or token.
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  message:
                    type: string
        '500':
          description: Unexpected server error
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  message:
                    type: string

  /v2/feedback:
    post:
      tags:
        - v2
      summary: Create feedback
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/CreateFeedbackV2"
      responses:
        "200":
          description: Feedback created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
        "400":
          description: Bad request
        "500":
          description: Server error

  /v2/feedback/{id}:
    get:
      tags:
        - v2
      summary: Get feedback
      parameters:
        - in: path
          name: id
          required: true
          schema:
            type: string
            format: uuid
          description: Feedback ID
      responses:
        "200":
          description: Feedback retrieved successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
        "404":
          description: Feedback not found

    patch:
      tags:
        - v2
      summary: Update feedback
      parameters:
        - in: path
          name: id
          required: true
          schema:
            type: string
            format: uuid
          description: Feedback ID
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/UpdateFeedbackV2"
      responses:
        "200":
          description: Feedback updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/FeedbackV2"
        "400":
          description: Bad request
        "500":
          description: Server error

  /v2/runs/{id}:
    get:
      tags:
        - v2
      summary: Get a run
      parameters:
        - in: path
          name: id
          required: true
          schema:
            type: string
            format: uuid
          description: The ID of the run to retrieve
      responses:
        "200":
          description: Run retrieved successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/RunV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/RunV2"
        "404":
          description: Run not found

  /v2/test-results:
    get:
      tags:
        - v2
      summary: Get test results
      parameters:
        - name: pipelineId
          in: query
          description: The ID of the pipeline
          schema:
            type: string
        - name: pipelineSlug
          in: query
          description: The slug of the pipeline
          schema:
            type: string
        - name: createdAt
          in: query
          style: deepObject
          explode: true
          schema:
            $ref: '#/components/schemas/SearchableUnixSecondsInput'
          description: Criteria to search by creation date
        - name: metadata
          in: query
          style: deepObject
          explode: true
          schema:
            $ref: '#/components/schemas/FilterableMetadataInput'
          description: Criteria to search by metadata

      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/TestResultV2'
                required:
                  - data
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/TestResultV2'
                required:
                  - data
    
  /v2/test-cases:
    get:
      tags:
        - v2
      summary: Get test cases
      parameters:
        - in: query
          name: pipelineId
          schema:
            type: string
            format: uuid
          description: The ID of the Pipeline to retrieve test cases for
        - in: query
          name: pipelineSlug
          schema:
            type: string
          description: The slug of the Pipeline to retrieve test cases for
      responses:
        "200":
          description: Test cases retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/TestCaseV2"
                required:
                  - data
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/TestCaseV2"
                required:
                  - data

  /v2/test-cases/{id}:
    get:
      tags:
        - v2
      summary: Get a test case
      parameters:
        - in: path
          name: id
          required: true
          schema:
            type: string
            format: uuid
          description: The ID of the test case to retrieve
      responses:
        "200":
          description: Test case retrieved successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/TestCaseV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/TestCaseV2"
        "404":
          description: Test case not found

  /v2/pipelines:
    get:
      tags:
        - v2
      summary: Get pipelines
      parameters:
        - in: query
          name: label
          schema:
            type: string
          description: The label to filter pipelines by
        - in: query
          name: slug
          style: deepObject
          explode: true
          description: The slug to filter pipelines by
          schema:
            $ref: "#/components/schemas/SearchableStringInput"
        - in: query
          name: folderId
          schema:
            type: string
            nullable: true
          description: The folder ID to filter pipelines by. Null filters to root folder. Absent will return all folders.
      responses:
        "200":
          description: Pipelines retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/PipelineV2"
                required:
                  - data
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  pipelines:
                    type: array
                    items:
                      $ref: "#/components/schemas/PipelineV2"
                required:
                  - data

  /v2/folders:
    get:
      tags:
        - v2
      summary: Get folders
      parameters:
        - in: query
          name: parentFolderId
          schema:
            type: string
            nullable: true
          description: The parent folder ID to filter folders by. Null filters to root folder. Absent will return all folders.
      responses:
        "200":
          description: Folders retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/FolderV2"
                required:
                  - data
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/FolderV2"
                required:
                  - data

  /v2/folders/{id}:
    get:
      tags:
        - v2
      summary: Get a folder
      parameters:
        - in: path
          name: id
          required: true
          schema:
            type: string
            format: uuid
          description: The ID of the folder to retrieve
      responses:
        "200":
          description: Folder retrieved successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/FolderV2"
            application/json; charset=utf-8:
              schema:
                $ref: "#/components/schemas/FolderV2"
        "404":
          description: Folder not found

  /v2/evaluators:
    get:
      tags:
        - v2
      summary: Get evaluators
      description: Get evaluators for a given pipeline
      parameters:
        - in: query
          name: pipelineId
          schema:
            type: string
            nullable: true
            format: uuid
          description: The ID of the Pipeline to retrieve evaluators for. Use "null" to retrieve the organization's evaluator templates.
        - in: query
          name: pipelineSlug
          schema:
            type: string
            nullable: true
          description: Or, the slug of the Pipeline to retrieve evaluators for
      responses:
        "200":
          description: Evaluators retrieved successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/EvaluatorV2"
                required:
                  - data
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: "#/components/schemas/EvaluatorV2"
                required:
                  - data
        "404":
          description: Pipeline not found

  /v2/evaluations/bulk:
    post:
      tags:
        - v2
      summary: Bulk create evaluations
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                data:
                  type: array
                  items:
                    $ref: "#/components/schemas/CreateEvaluationV2"
              required:
                - data
      responses:
        "200":
          description: Evaluations created successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  count:
                    type: number
                required:
                  - count
            application/json; charset=utf-8:
              schema:
                type: object
                properties:
                  count:
                    type: number
                required:
                  - count
        "400":
          description: Bad request
        "500":
          description: Server error
    
components:
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer

  schemas:
    UnixSeconds:
      type: number
      format: float
      description: Timestamp in seconds since the UNIX epoch. Can be transformed into a Date object.

    UnixSecondsNullable:
      type: number
      format: float
      nullable: true
      description: Timestamp in seconds since the UNIX epoch. Can be transformed into a Date object.
  
    SearchableUnixSecondsInput:
      oneOf:
        - $ref: '#/components/schemas/UnixSeconds'
        - type: object
          properties:
            gt:
              $ref: '#/components/schemas/UnixSeconds'
              description: Date should be greater than this value.
            gte:
              $ref: '#/components/schemas/UnixSeconds'
              description: Date should be greater than or equal to this value.
            lt:
              $ref: '#/components/schemas/UnixSeconds'
              description: Date should be less than this value.
            lte:
              $ref: '#/components/schemas/UnixSeconds'
              description: Date should be less than or equal to this value.

    SearchableStringInput:
      oneOf:
        - type: string
        - type: object
          properties:
            contains: 
              type: string
            search:
              type: string
              description: For Postgres full text search
            startsWith:
              type: string
            endsWith:
              type: string
            in:
              type: array
              items:
                type: string
            notIn:
              type: array
              items:
                type: string
            mode:
              type: string
              enum:
                - insensitive
                - default

    FilterableMetadataInput:
      type: object
      description: 'Metadata filters look like { prompt: { contains: "My prompt" } }'
      additionalProperties:
        type: object
        properties:
          exists:
            type: boolean
            description: Specifies if the metadata key exists.
          contains:
            type: string
            description: The metadata value contains this string.
          equals:
            oneOf:
              - type: string
              - type: number
            description: The metadata value is equal to this value.
          gt:
            oneOf:
              - type: string
              - type: number
            description: The metadata value is greater than this value.
          gte:
            oneOf:
              - type: string
              - type: number
            description: The metadata value is greater than or equal to this value.
          lt:
            oneOf:
              - type: string
              - type: number
            description: The metadata value is less than this value.
          lte:
            oneOf:
              - type: string
              - type: number
            description: The metadata value is less than or equal to this value.

    MetadataValueObject:
      type: object
      required:
        - type
      properties:
        type:
          type: string
      oneOf:
        - $ref: '#/components/schemas/UrlType'
        - $ref: '#/components/schemas/StringType'
        - $ref: '#/components/schemas/BooleanType'
      discriminator:
        propertyName: type
        mapping:
          url: '#/components/schemas/UrlType'
          string: '#/components/schemas/StringType'
          boolean: '#/components/schemas/BooleanType'

    UrlType:
      type: object
      required:
        - type
        - url
        - text
      properties:
        type:
          type: string
          enum:
            - url
        url:
          type: string
          format: uri
        text:
          type: string

    StringType:
      type: object
      required:
        - type
        - value
      properties:
        type:
          type: string
          enum:
            - string
        value:
          type: string

    BooleanType:
      type: object
      required:
        - type
        - value
      properties:
        type:
          type: string
          enum:
            - boolean
        value:
          type: boolean

    CreateEvaluationV2:
      type: object
      properties:
        note: 
          type: string
          description: Optionally add a note to the evaluation
        evaluatorId:
          type: string
          format: uuid
          description: The ID of the evaluator. The evaluator and run must be in the same pipeline.
        runId:
          type: string
          format: uuid
          description: The ID of the run. The evaluator and run must be in the same pipeline.
        evalLabel:
          type: string
          description: If the evaluator output type is an enum, the label of the enum value.
        evalValue:
          type: number
          description: If the evaluator output type is a percentage, a number between 0 and 1 representing the percentage.
      required:
        - evaluatorId
        - runId

    CreateFeedbackV2:
      type: object
      properties:
        pipelineRunId:
          type: string
          format: uuid
          description: The unique identifier for the pipeline run
        recordedTime:
          $ref: '#/components/schemas/UnixSeconds'
        score:
          type: number
          format: double
          minimum: 0
          maximum: 1
          description: The score of the feedback, ranging from 0 to 1
        details:
          type: string
          nullable: true
          description: Optional details about the feedback
      required:
        - pipelineRunId
        - recordedTime
        - score

    UpdateFeedbackV2:
      type: object
      properties:
        score:
          type: number
          format: double
          minimum: 0
          maximum: 1
          description: The score of the feedback, ranging from 0 to 1
        details:
          type: string
          nullable: true
          description: Optional details about the feedback

    FeedbackV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The unique identifier for the feedback
        pipelineRunId:
          type: string
          format: uuid
          description: The unique identifier for the pipeline run
        score:
          type: number
          format: double
          minimum: 0
          maximum: 1
          description: The score of the feedback, ranging from 0 to 1
        details:
          type: string
          nullable: true
          description: Optional details about the feedback
        recordedTime:
          $ref: '#/components/schemas/UnixSeconds'
      required:
        - id
        - pipelineRunId
        - score
        - recordedTime

    ExpandedTestResult:
      allOf:
        - $ref: '#/components/schemas/TestResult'
        - type: object
          properties:
            pipeline:
              $ref: '#/components/schemas/ExpandedPipeline'
            runs:
              type: array
              items:
                $ref: '#/components/schemas/ExpandedTestRun'

    ExpandedTestRun:
      allOf:
        - $ref: '#/components/schemas/TestRun'
        - type: object
          properties:
            full:
              $ref: '#/components/schemas/FullRun'
            steps:
              type: array
              items:
                $ref: '#/components/schemas/ResolvedStepRun'
            case:
              $ref: '#/components/schemas/TestCase'
            evaluations:
              type: array
              items:
                $ref: '#/components/schemas/TestEvaluation'

    FullRun:
      type: object
      required:
        - pipelineRunId
        - pipelineId
        - organizationId
        - startTime
        - endTime
      properties:
        pipelineRunId:
          type: string
          format: uuid
        pipelineId:
          type: string
          format: uuid
        organizationId:
          type: string
          format: uuid
        startTime:
          type: string
          format: datetime
        endTime:
          type: string
          format: datetime
        cost:
          type: number
          minimum: 0
          nullable: true
        elapsed:
          type: number
          minimum: 0
          nullable: true
        feedback:
          type: number
          minimum: 0
          maximum: 1
          nullable: true
        lastInvocation:
          type: string
          nullable: true
        inputs:
          type: object
          additionalProperties: true
          nullable: true
        outputs:
          type: object
          additionalProperties: true          
          nullable: true
        renderHTMLKey:
          type: string
          nullable: true
        metadata:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/MetadataValueObject'
          nullable: true

    TestRun:
      type: object
      required:
        - id
        - createdAt
        - updatedAt
        - caseId
        - resultId
      properties:
        id:
          type: string
          format: uuid
        createdAt:
          type: string
          format: datetime
        updatedAt:
          type: string
          format: datetime
        caseId:
          type: string
          format: uuid
        resultId:
          type: string
          format: uuid

    ExpandedPipeline:
      allOf:
        - $ref: '#/components/schemas/Pipeline'
        - type: object
          properties:
            evaluators:
              type: array
              items:
                $ref: '#/components/schemas/TestEvaluator'

    TestEvaluator:
      type: object
      required:
        - id
        - createdAt
        - updatedAt
        - name
        - options
        - pipelineId
        - who
        - valueType
      properties:
        id:
          type: string
          format: uuid
        createdAt:
          type: string
          format: datetime
        updatedAt:
          type: string
          format: datetime
        archivedAt:
          type: string
          format: datetime
          nullable: true
        icon:
          type: string
          nullable: true
        name:
          type: string
        options:
          type: array
          items: {} 
          nullable: true
        # TODO: Enum values were causing runtime check issues with the Python
        # SDK. Figure out what is causing the issue at a later point.
        aiModel:
          type: string
          nullable: true
        pipelineId:
          type: string
        processorId:
          type: string
          nullable: true
        heuristicFn:
          type: string
          nullable: true
        aiPromptFormat:
          type: string
          nullable: true
        humanPrompt:
          type: string
          nullable: true
        who:
          type: string
        valueType:
          type: string

    RunRequest:
      type: object
      properties:
        id:
          type: string
          format: uuid
        collectionMethod:
          type: string
          description: The method used to collect the run
          enum:
            - manual
            - runner
        slug:
          type: string
        previousRunId:
          type: string
          format: uuid
          nullable: true
        metadata:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/MetadataValueObject'
          nullable: true
        stepRuns:
          type: array
          items:
            $ref: '#/components/schemas/StepRun'
      required:
        - stepRuns
        - id

    StepRun:
      type: object
      properties:
        providerName:
          type: string
        invocation:
          type: string
        modelParams:
          type: object
          additionalProperties: true
        inputs:
          type: object
          additionalProperties: true
        outputs:
          type: object
          additionalProperties: true
        context: 
          type: object
          properties:
            userId:
              type: string
            render:
              type: object
              properties:
                type:
                  type: string
                key:
                  type: string
              required:
                - type
            metadata:
              type: object
              additionalProperties:
                $ref: '#/components/schemas/MetadataValueObject'
              nullable: true
        elapsedTime:
          type: integer
        startTime:
          type: string
          format: date-time
        endTime:
          type: string
          format: date-time
      required:
        - providerName
        - invocation
        - modelParams
        - inputs
        - outputs
        - elapsedTime
        - startTime
        - endTime

    ResolvedStepRun:
      type: object
      properties:
        id:
          type: string
          format: uuid
        organizationId:
          type: string
          format: uuid
        providerName:
          type: string
        invocation:
          type: string
        modelParams:
          type: object
          additionalProperties: true
        inputs:
          type: object
          additionalProperties: true
        outputs:
          type: object
          additionalProperties: true
        startTime:
          type: string
          format: datetime
        endTime:
          type: string
          format: datetime
        pipelineRunId:
          type: string
          format: uuid

    RunResponse:
      type: object
      properties:
        pipelineRunId:
          type: string
          format: uuid
      example:
        pipelineRunId: 13448D47-BD5D-4664-B6C3-0CB07D8CF2D2

    FeedbackRequest:
      type: object
      properties:
        pipelineRunId:
          type: string
          format: uuid
        score:
          type: number
          format: float
          minimum: 0
          maximum: 1.0
        recordedTime:
          type: string
          format: date-time
        details:
          type: string
          nullable: true
      required:
        - pipelineRunId
        - score
        - recordedTime
      additionalProperties: false

    FeedbackResponse:
      type: object
      properties:
        message:
          type: string
          example: Success
      required:
        - message
      additionalProperties: false

    RunV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the run
        stepRuns:
          type: array
          items:
            type: object
            properties:
              stepRunId:
                type: string
                format: uuid
                description: The ID of the step run
              stepId:
                type: string
                format: uuid
                description: The ID of the step
              providerName:
                type: string
                description: The name of the provider
              invocation:
                type: string
                description: The name of the method being invoked
              order:
                type: integer
                description: The order of the step run. The last step has the highest "order" number. 
              modelParams:
                type: object
                description: The parameters used by the model
              inputs:
                type: object
                description: The inputs of the run
              outputs:
                type: object
                description: The outputs of the run
              elapsedTime:
                type: integer
                description: The milliseconds to complete the run
              startTime:
                $ref: '#/components/schemas/UnixSeconds'
              endTime:
                $ref: '#/components/schemas/UnixSeconds'
              cost:
                type: number
                description: The cost of the run in USD
              metadata:
                type: object
                additionalProperties:
                  $ref: '#/components/schemas/MetadataValueObject'
                nullable: true
            required:
              - stepRunId
              - stepId
              - providerName
              - invocation
              - order
              - modelParams
              - inputs
              - outputs
              - elapsedTime
              - startTime
              - endTime
              - cost
      required:
        - id
        - stepRuns

    TestCase:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the test case
        createdAt:
          type: string
          format: date-time
          description: The date and time when the test case was created
        archivedAt:
          type: string
          nullable: true
          format: date-time
          description: The date and time when the test case was archived, can be null if the test case has not been archived
        updatedAt:
          type: string
          format: date-time
          description: The date and time when the test case was last updated
        expectedOutputs:
          type: object
          nullable: true
          description: The expected outputs for the test case
        inputs:
          type: object
          additionalProperties: true
          description: The input data for the test case as a JSON object
        name:
          type: string
          description: The name of the test case
        pipelineId:
          type: string
          format: uuid
          description: The ID of the pipeline that the test case belongs to
      required:
        - id
        - createdAt
        - updatedAt
        - inputs
        - name
        - pipelineId

    CreateSingleTestCase:
      type: object
      properties:
        pipelineSlug:
          type: string
          description: Slug for the pipeline
        name:
          type: string
          description: The name of the test case
        inputs:
          type: object
          additionalProperties: true
          description: The input data for the test case as a JSON object
        expectedOutputs:
          type: object
          nullable: true
          description: The expected outputs for the test case as a JSON object

    CreateMultipleTestCases:
      type: object
      properties:
        pipelineSlug:
          type: string
          description: Slug for the pipeline
        testCases:
          type: array
          items:
            type: object
            properties:
              name:
                type: string
                minLength: 1
                description: Name of the test case.
              inputs:
                type: object
                description: Input for the test case. Must be a valid JSON object and not an array.
                additionalProperties: true
              expectedOutputs:
                type: object
                description: Expected outputs for the test case
            required:
            - name
            - inputs
            - expectedOutputs

    UpdateTestCase:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: ID of the test case to update
        name:
          type: string
          nullable: true
          description: Name of the test case
        inputs:
          type: object
          nullable: true
          description: Inputs for the test case
        expectedOutputs:
          type: object
          nullable: true
          description: Expected outputs for the test case
        archived:
          type: boolean
          nullable: true
          description: Archive status for the test case
      required:
        - id

    Pipeline:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the pipeline
        createdAt:
          type: string
          format: date-time
          description: The date and time when the pipeline was created
        updatedAt:
          type: string
          format: date-time
          description: The date and time when the pipeline was last updated
        archivedAt:
          type: string
          format: date-time
          nullable: true
          description: The date and time when the pipeline was archived, can be null if the pipeline has not been archived
        labels:
          type: array
          items:
            type: string
          description: The labels attached to the pipeline
        displayName:
          type: string
          description: The name of the pipeline
          nullable: true
        slug:
          type: string
          description: The slug of the pipeline
        organizationId:
          type: string
          description: The ID of the organization that owns the pipeline
        branch:
          type: string
          nullable: true
          description: The branch that the pipeline is associated with
      required:
        - id
        - createdAt
        - updatedAt
        - labels
        - slug
        - organizationId

    TestResult:
      type: object
      required:
        - id
        - createdAt
        - updatedAt
        - pipelineId
      properties:
        id:
          type: string
          format: uuid
          description: The unique identifier for the test result.
        createdAt:
          type: string
          format: date-time
          description: The date and time the test result was created.
        updatedAt:
          type: string
          format: date-time
          description: The date and time the test result was last updated.
        pipelineId:
          type: string
          format: uuid
          description: The unique identifier for the pipeline associated with the test result.
        branch:
          type: string
          description: The branch that the test result was created from
          nullable: true
        commit:
          type: string
          description: The commit that the test result was created from
          nullable: true
        metadata:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/MetadataValueObject'
          nullable: true
        name:
          type: string
          description: The name of the test result
          nullable: true

    TestEvaluation:
      type: object
      required:
        - id
        - createdAt
        - updatedAt
        - isPending
        - evaluatorId
        - runId
        - billingGpt4InputTokens
        - billingGpt4OutputTokens
        - billingGpt35InputTokens
        - billingGpt35OutputTokens
      properties:
        id:
          type: string
          format: uuid
        createdAt:
          type: string
          format: datetime
        updatedAt:
          type: string
          format: datetime
        isPending:
          type: boolean
        debug:
          type: object
          nullable: true
        evaluatorId:
          type: string
          format: uuid
        runId:
          type: string
          format: uuid
        evalLabel:
          type: string
          nullable: true
        evalValue:
          type: number
          nullable: true
        manualCreatedByEmail:
          type: string
          nullable: true
        billingGpt4InputTokens:
          type: integer
        billingGpt4OutputTokens:
          type: integer
        billingGpt35InputTokens:
          type: integer
        billingGpt35OutputTokens:
          type: integer

    PipelineV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the pipeline
        createdAt:
          $ref: '#/components/schemas/UnixSeconds'
        updatedAt:
          $ref: '#/components/schemas/UnixSeconds'
        archivedAt:
          $ref: '#/components/schemas/UnixSecondsNullable'
        labels:
          type: array
          items:
            type: string
          description: The labels attached to the pipeline
        displayName:
          type: string
          description: The name of the pipeline
          nullable: true
        slug:
          type: string
          description: The slug of the pipeline
        organizationId:
          type: string
          description: The ID of the organization that owns the pipeline
        branch:
          type: string
          nullable: true
          description: The branch that the pipeline is associated with
        privateMemberId:
          type: string
          nullable: true
          description: If null, this is a team pipeline. If not null, this is a private pipeline for the specified member ID.
        privateUserEmail:
          type: string
          nullable: true
          description: If null, this is a team pipeline. If not null, this is a private pipeline for the specified email.
      required:
        - id
        - createdAt
        - updatedAt
        - archivedAt
        - labels
        - slug
        - branch
        - organizationId
        - privateMemberId
        - privateUserEmail

    TestResultV2:
      type: object
      required:
        - id
        - archivedAt
        - createdAt
        - updatedAt
        - pipelineId
      properties:
        id:
          type: string
          format: uuid
          description: The unique identifier for the test result.
        createdAt:
          $ref: '#/components/schemas/UnixSeconds'
        updatedAt:
          $ref: '#/components/schemas/UnixSeconds'
        archivedAt:
          $ref: '#/components/schemas/UnixSecondsNullable'
        pipelineId:
          type: string
          format: uuid
          description: The unique identifier for the pipeline associated with the test result.
        branch:
          type: string
          description: The branch that the test result was created from
          nullable: true
        commit:
          type: string
          description: The commit that the test result was created from
          nullable: true
        metadata:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/MetadataValueObject'
          nullable: true
        name:
          type: string
          description: The name of the test result
          nullable: true

    TestCaseV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the test case
        createdAt:
          $ref: '#/components/schemas/UnixSeconds'
        updatedAt:
          $ref: '#/components/schemas/UnixSeconds'
        archivedAt:
          $ref: '#/components/schemas/UnixSecondsNullable'
        expectedOutputs:
          type: object
          nullable: true
          description: The expected outputs for the test case
        inputs:
          type: object
          additionalProperties: true
          description: The input data for the test case as a JSON object
        name:
          type: string
          description: The name of the test case
        pipelineId:
          type: string
          format: uuid
          description: The ID of the pipeline that the test case belongs to
      required:
        - id
        - createdAt
        - updatedAt
        - archivedAt
        - inputs
        - name
        - pipelineId
    
    FolderV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the folder
        createdAt:
          $ref: '#/components/schemas/UnixSeconds'
        updatedAt:
          $ref: '#/components/schemas/UnixSeconds'
        name:
          type: string
          description: The name of the folder
        organizationId:
          type: string
          format: uuid
          description: The ID of the organization that owns the folder
        parentFolderId:
          type: string
          format: uuid
          nullable: true
          description: The ID of the parent folder
      required:
        - id
        - createdAt
        - updatedAt
        - name
        - organizationId
        - parentFolderId

    EvaluatorV2:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the evaluator
        createdAt:
          $ref: '#/components/schemas/UnixSeconds'
        updatedAt:
          $ref: '#/components/schemas/UnixSeconds'
        archivedAt:
          $ref: '#/components/schemas/UnixSecondsNullable'
        icon: 
          type: string
          nullable: true
        name:
          type: string
          description: The name of the evaluator
        options:
          type: array
          items: {}
          description: For evaluators with options scoring, the available options to choose from
          nullable: true
        aiModel:
          type: string
          description: For AI evaluators, the AI model to use
        pipelineId:
          type: string
          format: uuid
          description: The ID of the pipeline that the evaluator belongs to
          nullable: true
        processorId:
          type: string
          format: uuid
          description: The ID of the processor associated with the evaluator
          nullable: true
        organizationId:
          type: string
          format: uuid
          description: The ID of the organization that the evaluator belongs to
        templateDescription:
          type: string
          description: For evaluator templates, the description of the template
        heuristicFn:
          type: string
          description: For heuristic evaluators, the heuristic function to use
          nullable: true
        heuristicFnLanguage:
          type: string
          description: For heuristic evaluators, the coding language of the heuristic function (such as "JAVASCRIPT", "PYTHON")
        aiPromptFormat:
          type: string
          description: For AI evaluators, the prompt template that should be sent to the AI model
          nullable: true
        aiImageUrls:
          type: array
          items:
            type: string
          description: For AI image evaluators, the paths to the image URLs
        humanPrompt:
          type: string
          description: For human evaluators, the instructions for the human to follow
          nullable: true
        classifierValuePath:
          type: string
          description: For classification evaluators, the path to the predicted classification
          nullable: true
        classifierExpectedValuePath:
          type: string
          description: For classification evaluators, the path to the expected classification
          nullable: true
        multiClassOptions:
          type: array
          items:
            type: string
          description: For classification evaluators using multi-class evaluation, the available options to match with
        who:
          type: string
          description: The type of evaluator (such as "AI", "HEURISTIC", "HUMAN", "CLASSIFIER")
        valueType:
          type: string
          description: The scoring method used by the evaluator (such as "ENUM", "PERCENTAGE")
        runCondition:
          type: string
          description: The run condition of the evaluator (such as "TEST_PROD", "TEST", "PROD", "COMPARISON_2")
        prodEvalActive:
          deprecated: true
          type: boolean
          description: Use "samplingProbability" instead
        samplingProbability:
          type: number
          description: When optionally running on production data, the associated sampling probability of this evaluator (from 0 to 100)
          nullable: true
      required:
        - id
        - name
        - createdAt
        - updatedAt
        - who
        - valueType
        - runCondition