insta-integration-config-latest.json

{
  "$schema": "https://json-schema.org/draft/2019-09/schema",
  "title": "insta-integration configuration schema",
  "description": "Configuration to run integration tests with insta-integration",
  "type": "object",
  "properties": {
    "env": {
      "$ref": "#/$defs/EnvironmentVariables",
      "description": "Environment variables to be applied to all runs."
    },
    "services": {
      "type": "array",
      "description": "External services required to run integration tests",
      "items": {
        "type": "object",
        "properties": {
          "name": {
            "type": "string",
            "description": "Name of the service. Optionally, define the version of the service to run using pattern '<service>:<version>'.",
            "examples": ["kafka", "mysql", "postgres", "postgres:16.3"]
          },
          "data": {
            "type": "string",
            "description": "Folder path or link to startup files. For example, 'postgres/sql' folder with schema and table creation SQL files. Refer to insta-infra for possible startup files"
          },
          "env": {
            "$ref": "#/$defs/EnvironmentVariables",
            "description": "Environment variables for the service."
          }
        },
        "additionalProperties": false,
        "required": ["name"]
      }
    },
    "run": {
      "type": "array",
      "description": "Applications/jobs to run as part of the integration tests. Run in order.",
      "items": {
        "type": "object",
        "properties": {
          "command": {
            "type": "string",
            "description": "Bash command to run your application/job."
          },
          "commandWaitForFinish": {
            "type": "boolean",
            "description": "Whether to wait for the command to finish before running data generation/validation.",
            "default": true
          },
          "env": {
            "$ref": "#/$defs/EnvironmentVariables",
            "description": "Environment variables for your application/job."
          },
          "generateFirst": {
            "type": "boolean",
            "description": "Whether to run data generation before your application/job runs."
          },
          "test": {
            "type": "object",
            "description": "Data Caterer configuration for data generation and validation",
            "properties": {
              "env": {
                "$ref": "#/$defs/EnvironmentVariables",
                "description": "Environment variables for data-caterer Docker image. Check 'src/config.js' baseApplicationConf for potential environment variables to override."
              },
              "generation": {
                "type": "object",
                "description": "Define data sources along with schemas for data generation.",
                "propertyNames": {
                  "enum": [
                    "bigquery",
                    "cassandra",
                    "postgres",
                    "mysql",
                    "csv",
                    "delta",
                    "iceberg",
                    "json",
                    "orc",
                    "parquet",
                    "http",
                    "kafka",
                    "solace",
                    "rabbitmq"
                  ]
                },
                "additionalProperties": {
                  "type": "array",
                  "description": "Array of data generation tasks with the data source name as the key.",
                  "items": {
                    "type": "object",
                    "description": "Data generation task details with information about data source, number of records and schema.",
                    "properties": {
                      "name": {
                        "type": "string"
                      },
                      "options": {
                        "type": "object",
                        "description": "Data source configuration for data generation.",
                        "properties": {
                          "metadataSourceType": {
                            "description": "Type of metadata source for retrieval of schema information.",
                            "enum": [
                              "marquez",
                              "openApi",
                              "openMetadata",
                              "openDataContractStandard",
                              "dataContractCli"
                            ]
                          }
                        },
                        "allOf": [
                          {
                            "if": {
                              "properties": {
                                "metadataSourceType": {
                                  "const": "openApi"
                                }
                              },
                              "required": ["metadataSourceType"]
                            },
                            "then": {
                              "properties": {
                                "schemaLocation": {
                                  "description": "Location of OpenAPI spec file (can be a URL or file pathway).",
                                  "type": "string"
                                }
                              },
                              "required": ["schemaLocation"]
                            }
                          },
                          {
                            "if": {
                              "properties": {
                                "metadataSourceType": {
                                  "const": "openMetadata"
                                }
                              },
                              "required": ["metadataSourceType"]
                            },
                            "then": {
                              "properties": {
                                "tableFqn": {
                                  "description": "Table full qualified name."
                                }
                              }
                            }
                          },
                          {
                            "if": {
                              "properties": {
                                "metadataSourceType": {
                                  "const": "marquez"
                                }
                              },
                              "required": ["metadataSourceType"]
                            },
                            "then": {
                              "properties": {
                                "metadataSourceUrl": {
                                  "description": "Marquez URL."
                                },
                                "namespace": {
                                  "description": "Namespace to retrieve metadata from."
                                },
                                "dataset": {
                                  "description": "Dataset to retrieve metadata from."
                                }
                              },
                              "required": ["metadataSourceUrl", "namespace"]
                            }
                          },
                          {
                            "if": {
                              "properties": {
                                "metadataSourceType": {
                                  "enum": [
                                    "openDataContractStandard",
                                    "dataContractCli"
                                  ]
                                }
                              },
                              "required": ["metadataSourceType"]
                            },
                            "then": {
                              "properties": {
                                "dataContractFile": {
                                  "description": "Data contract file location."
                                }
                              },
                              "required": ["dataContractFile"]
                            }
                          }
                        ]
                      },
                      "count": {
                        "type": "object",
                        "description": "Define the number of records to generate.",
                        "properties": {
                          "records": {
                            "type": "integer",
                            "description": "Total number of records to generate."
                          },
                          "options": {
                            "description": "Random number of records defined by metadata in options.",
                            "$ref": "#/$defs/CountGeneratorOptions"
                          },
                          "perField": {
                            "type": "object",
                            "description": "Generate records per unique set of field values. For example, generate 5 transactions for each 'account_number'.",
                            "properties": {
                              "fieldNames": {
                                "type": "array",
                                "description": "Array of field names to generate records for each set of unique values.",
                                "items": {
                                  "type": "string",
                                  "description": "Field name found within schema fields."
                                }
                              },
                              "count": {
                                "type": "integer",
                                "description": "Static number of records per unique set of field values."
                              },
                              "options": {
                                "description": "Random number of records per unique set of field values.",
                                "$ref": "#/$defs/CountGeneratorOptions"
                              }
                            },
                            "additionalProperties": false
                          }
                        }
                      },
                      "fields": {
                        "type": "array",
                        "description": "Fields to generate.",
                        "items": {
                          "$ref": "#/$defs/SchemaField"
                        }
                      }
                    },
                    "additionalProperties": false
                  }
                },
                "allOf": [
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["postgres", "mysql"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "dbtable": {
                                  "description": "Name of the schema and table to connect to (<schema>.<table> format).",
                                  "type": "string",
                                  "pattern": "^(.+?)\\.(.+?)$"
                                }
                              },
                              "required": ["dbtable"]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  },
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["cassandra"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "keyspace": {
                                  "description": "Cassandra keyspace name."
                                },
                                "table": {
                                  "description": "Cassandra table name."
                                }
                              },
                              "required": ["keyspace", "table"]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  },
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["csv", "delta", "json", "orc", "parquet"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "partitions": {
                                  "description": "Number of file partitions to create."
                                },
                                "path": {
                                  "description": "Path to file."
                                },
                                "partitionBy": {
                                  "description": "Field name(s) to partition dataset by (comma separated)."
                                }
                              },
                              "required": ["path"]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  },
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["iceberg"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "catalogType": {
                                  "description": "Type of Iceberg catalog.",
                                  "enum": [
                                    "hive",
                                    "hadoop",
                                    "rest",
                                    "glue",
                                    "jdbc",
                                    "nessie"
                                  ]
                                },
                                "catalogUri": {
                                  "description": "URI for catalog."
                                },
                                "path": {
                                  "description": "Warehouse path for catalog."
                                },
                                "table": {
                                  "description": "Table name of Iceberg dataset."
                                }
                              },
                              "required": [
                                "catalogType",
                                "catalogUri",
                                "path",
                                "table"
                              ]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  },
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["kafka"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "topic": {
                                  "description": "Topic name."
                                },
                                "rowsPerSecond": {
                                  "description": "Number of records to create per second."
                                }
                              },
                              "required": ["topic"]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  },
                  {
                    "if": {
                      "propertyNames": {
                        "enum": ["solace"]
                      }
                    },
                    "then": {
                      "additionalProperties": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "options": {
                              "type": "object",
                              "properties": {
                                "destinationName": {
                                  "description": "Name of the topic/queue."
                                },
                                "rowsPerSecond": {
                                  "description": "Number of records to create per second."
                                }
                              },
                              "required": ["destinationName"]
                            }
                          },
                          "required": ["options"]
                        }
                      }
                    }
                  }
                ]
              },
              "mount": {
                "description": "Add volume mount for data-caterer Docker image.",
                "type": "array",
                "items": {
                  "type": "string",
                  "pattern": ".*:.*"
                }
              },
              "relationship": {
                "type": "object",
                "description": "Define any relationships between data generation data sources. For example, 'account_number' field being the same across two Postgres tables accounts and balances.",
                "patternProperties": {
                  "^[^|]+\\|\\|[^|]+(\\|\\|[^|]+)?$": {
                    "type": "array",
                    "items": {
                      "type": "string",
                      "description": "Uses the pattern: '<name of generation>.<field name>' or '<name of data source>.<name of generation>.<field name>'",
                      "pattern": "^[^|]+\\|\\|[^|]+(\\|\\|[^|]+)?$",
                      "examples": ["postgres_transaction||account_number", "http||POST/pets||body.id"]
                    }
                  }
                },
                "additionalProperties": false
              },
              "validation": {
                "type": "object",
                "description": "Define data sources along with validation rules to run after data generation and your application/job.",
                "propertyNames": {
                  "enum": [
                    "bigquery",
                    "cassandra",
                    "postgres",
                    "mysql",
                    "csv",
                    "delta",
                    "iceberg",
                    "json",
                    "orc",
                    "parquet",
                    "http"
                  ]
                },
                "additionalProperties": {
                  "type": "array",
                  "description": "Array of data validation tasks with the data source name as the key.",
                  "items": {
                    "$ref": "#/$defs/DataCatererBaseValidation"
                  }
                }
              }
            },
            "additionalProperties": false
          }
        },
        "additionalProperties": false
      }
    }
  },
  "additionalProperties": false,
  "$defs": {
    "EnvironmentVariables": {
      "type": "object",
      "additionalProperties": {
        "type": "string"
      }
    },
    "CountGeneratorOptions": {
      "type": "object",
      "propertyNames": {
        "enum": ["sql", "min", "max", "stddev", "mean", "oneOf"]
      },
      "allOf": [
        {
          "if": {
            "required": ["oneOf"]
          },
          "then": {
            "properties": {
              "oneOf": {
                "oneOf": [
                  {
                    "type": "array",
                    "items": {
                      "type": "integer"
                    }
                  },
                  {
                    "type": "array",
                    "items": {
                      "type": "string",
                      "pattern": "^0|[1-9]\\d*->(0|[1-9]\\d*|\\d+\\.\\d+)$"
                    }
                  }
                ]
              }
            }
          }
        }
      ]
    },
    "SchemaFieldDataTypes": {
      "enum": [
        "string",
        "integer",
        "int",
        "long",
        "double",
        "short",
        "decimal",
        "float",
        "date",
        "timestamp",
        "boolean",
        "binary",
        "byte",
        "array",
        "struct"
      ]
    },
    "SchemaFieldBaseOptions": {
      "type": "object",
      "properties": {
        "enabledEdgeCase": {
          "type": "boolean",
          "description": "Enable generating edge case values for data type.",
          "default": false
        },
        "edgeCaseProb": {
          "type": "number",
          "description": "Probability of generating edge case values. Range from 0-1.",
          "minimum": 0,
          "maximum": 1,
          "default": 0
        },
        "isUnique": {
          "type": "boolean",
          "description": "Generate only unique values.",
          "default": false
        },
        "seed": {
          "type": "number",
          "description": "Seed for generating consistent random values.",
          "default": -1
        },
        "sql": {
          "type": "string",
          "description": "Spark SQL expression for generating data."
        },
        "oneOf": {
          "type": "array",
          "description": "Generated values will be one of the defined values."
        },
        "omit": {
          "type": "boolean",
          "description": "Exclude the field from the final output. Can be used for intermediate data generation.",
          "default": false
        }
      }
    },
    "SchemaNullOptions": {
      "type": "object",
      "properties": {
        "enableNull": {
          "type": "boolean",
          "description": "Enable generation of null values.",
          "default": false
        },
        "nullProbability": {
          "type": "number",
          "description": "Probability of generating null values. Range from 0-1.",
          "minimum": 0,
          "maximum": 1,
          "default": 0
        }
      }
    },
    "SchemaBaseNumberOptions": {
      "type": "object",
      "properties": {
        "stddev": {
          "type": "number",
          "description": "Standard deviation of generated values.",
          "default": 1.0
        },
        "mean": {
          "type": "number",
          "description": "Mean of generated values.",
          "default": 500.0
        },
        "distribution": {
          "type": "string",
          "description": "Type of distribution values follow.",
          "enum": ["normal", "exponential"]
        },
        "distributionRateParam": {
          "type": "number",
          "description": "If distribution is `exponential`, rate parameter to adjust exponential distribution."
        },
        "incremental": {
          "type": "number",
          "description": "Generate incremental numbers. By default, starts at 1. You can define a starting number.",
          "default": 1
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        }
      ]
    },
    "SchemaStringOptions": {
      "type": "object",
      "properties": {
        "minLen": {
          "type": "integer",
          "description": "Minimum length of generated values.",
          "default": 1
        },
        "maxLen": {
          "type": "integer",
          "description": "Maximum length of generated values.",
          "default": 10
        },
        "expression": {
          "type": "string",
          "description": "Faker expression to generate values.",
          "examples": ["#{Name.name}"]
        },
        "regex": {
          "type": "string",
          "description": "Regex for generating values.",
          "examples": ["ACC[0-9]{8}"]
        },
        "uuid": {
          "type": "string",
          "description": "Generate UUID values. If non-empty, you can define another column name to generate UUID values from another columns value."
        },
        "incremental": {
          "type": "number",
          "description": "Generate incremental numbers. By default, starts at 1. You can define a starting number.",
          "default": 1
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        },
        {
          "$ref": "#/$defs/SchemaNullOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaIntegerOptions": {
      "type": "object",
      "properties": {
        "min": {
          "type": "integer",
          "description": "Minimum generated value.",
          "default": 0
        },
        "max": {
          "type": "integer",
          "description": "Maximum generated value.",
          "default": 1000
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaBaseNumberOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaNumberOptions": {
      "type": "object",
      "properties": {
        "min": {
          "type": "number",
          "description": "Minimum generated value.",
          "default": 0.0
        },
        "max": {
          "type": "number",
          "description": "Maximum generated value.",
          "default": 1000.0
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        },
        {
          "$ref": "#/$defs/SchemaBaseNumberOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaDateOptions": {
      "type": "object",
      "properties": {
        "min": {
          "type": "string",
          "description": "Maximum generated value."
        },
        "max": {
          "type": "string",
          "description": "Maximum generated value."
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        },
        {
          "$ref": "#/$defs/SchemaNullOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaBinaryOptions": {
      "type": "object",
      "properties": {
        "minLen": {
          "type": "number",
          "description": "Minimum length of generated values."
        },
        "maxLen": {
          "type": "number",
          "description": "Maximum length of generated values."
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        },
        {
          "$ref": "#/$defs/SchemaNullOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaArrayOptions": {
      "type": "object",
      "properties": {
        "arrayMinLen": {
          "type": "number",
          "description": "Minimum generated array length."
        },
        "arrayMaxLen": {
          "type": "number",
          "description": "Maximum generated array length."
        },
        "arrayType": {
          "type": "string",
          "description": "Data type of array values.",
          "$ref": "#/$defs/SchemaFieldDataTypes"
        }
      },
      "allOf": [
        {
          "$ref": "#/$defs/SchemaFieldBaseOptions"
        },
        {
          "$ref": "#/$defs/SchemaNullOptions"
        }
      ],
      "unevaluatedProperties": false
    },
    "SchemaField": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Name of the field."
        },
        "type": {
          "type": "string",
          "description": "Data type of the field.",
          "default": "string",
          "allOf": [
            {
              "$ref": "#/$defs/SchemaFieldDataTypes"
            }
          ]
        },
        "static": {
          "type": "string",
          "description": "Field is only equal to a static value."
        },
        "options": {
          "type": "object"
        }
      },
      "required": ["name"],
      "allOf": [
        {
          "if": {
            "properties": {
              "type": false
            }
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaStringOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "const": "string"
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaStringOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "anyOf": [
                  {
                    "const": "integer"
                  },
                  {
                    "const": "int"
                  }
                ]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaIntegerOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "anyOf": [
                  {
                    "const": "long"
                  },
                  {
                    "const": "short"
                  },
                  {
                    "const": "decimal"
                  },
                  {
                    "const": "double"
                  },
                  {
                    "const": "float"
                  }
                ]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaNumberOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "anyOf": [
                  {
                    "const": "date"
                  },
                  {
                    "const": "timestamp"
                  }
                ]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaDateOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "const": "binary"
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaBinaryOptions"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "const": "array"
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaArrayOptions"
              },
              "fields": {
                "type": "array",
                "description": "Inner structure of fields.",
                "items": {
                  "$ref": "#/$defs/SchemaField"
                }
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "const": "struct"
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "options": {
                "$ref": "#/$defs/SchemaFieldBaseOptions"
              },
              "fields": {
                "type": "array",
                "description": "Inner structure of fields.",
                "items": {
                  "$ref": "#/$defs/SchemaField"
                }
              }
            }
          }
        }
      ],
      "unevaluatedProperties": false
    },
    "DataCatererValidationRule": {
      "type": "object",
      "description": "Data validation rule.",
      "anyOf": [
        {
          "$ref": "#/$defs/FieldValidations"
        },
        {
          "$ref": "#/$defs/BasicValidation"
        },
        {
          "$ref": "#/$defs/FieldNameValidation"
        },
        {
          "$ref": "#/$defs/AggregationValidation"
        },
        {
          "$ref": "#/$defs/UpstreamValidation"
        }
      ],
      "unevaluatedProperties": false
    },
    "DataCatererBaseValidation": {
      "type": "object",
      "properties": {
        "options": {
          "type": "object",
          "description": "Data source configuration details."
        },
        "validations": {
          "type": "array",
          "description": "Array of data validation rules.",
          "items": {
            "$ref": "#/$defs/DataCatererValidationRule"
          },
          "minItems": 1
        },
        "waitCondition": {
          "type": "object"
        },
        "unevaluatedProperties": false
      }
    },
    "ValidationDescription": {
      "type": "string",
      "description": "Description of the validation rule. Helpful to provide additional context."
    },
    "ValidationErrorThreshold": {
      "type": "number",
      "description": "Define the number of records or percentage of records that do not meet the validation rule before marking the validation as failed."
    },
    "ValidationPreFilterExpression": {
      "type": "string",
      "description": "SQL expression that is run to filter data before running the validation."
    },
    "ValidationExpression": {
      "type": "string",
      "description": "SQL expression to define validation. Check https://spark.apache.org/docs/latest/api/sql for reference."
    },
    "ValidationSelectExpression": {
      "type": "string",
      "description": "SQL expression used to apply to fields before running validations."
    },
    "ValidationAdditionalProperties": {
      "type": "object",
      "properties": {
        "description": {
          "$ref": "#/$defs/ValidationDescription"
        },
        "errorThreshold": {
          "$ref": "#/$defs/ValidationErrorThreshold"
        },
        "preFilterExpr": {
          "$ref": "#/$defs/ValidationPreFilterExpression"
        }
      }
    },
    "ValidationAggregateExpression": {
      "type": "string",
      "description": "SQL expression to define validation after aggregation. Check https://spark.apache.org/docs/latest/api/sql for reference."
    },
    "ValidationAggregateField": {
      "type": "string",
      "description": "Field name to run aggregation on."
    },
    "ValidationAggregateType": {
      "type": "string",
      "description": "Type of aggregation to run. Can be either: sum, avg, max, min, mean, stddev, count.",
      "enum": ["sum", "avg", "max", "min", "mean", "stddev", "count"]
    },
    "ValidationAggregateGroupByFields": {
      "type": "array",
      "description": "Field names to group by for aggregation calculation.",
      "items": {
        "type": "string"
      }
    },
    "ValidationUpstreamJoinFields": {
      "type": "array",
      "description": "Field names to run join operation on.",
      "items": {
        "type": "string"
      },
      "minItems": 1
    },
    "ValidationUpstreamJoinType": {
      "type": "string",
      "description": "Type of join to run.",
      "enum": [
        "inner",
        "outer",
        "full",
        "full_outer",
        "left",
        "left_outer",
        "right",
        "right_outer",
        "left_semi",
        "semi",
        "left_anti",
        "anti",
        "cross"
      ],
      "default": "inner"
    },
    "ValidationUpstreamDataSource": {
      "type": "string",
      "description": "Name of the upstream data source."
    },
    "ValidationUpstreamReadOptions": {
      "type": "object",
      "description": "Additional configuration to connect to upstream data source."
    },
    "ValidationUpstreamNestedValidations": {
      "type": "array",
      "description": "Validation to run after join with upstream data source.",
      "items": {
        "$ref": "#/$defs/DataCatererValidationRule"
      },
      "minItems": 1
    },
    "ValidationFieldNameType": {
      "type": "string",
      "description": "Type of field name validation to run.",
      "enum": [
        "fieldCountEqual",
        "fieldCountBetween",
        "fieldNameMatchOrder",
        "fieldNameMatchSet"
      ]
    },
    "ValidationFieldNameCount": {
      "type": "integer",
      "description": "Expected number of field names."
    },
    "ValidationFieldNameMaxCount": {
      "type": "integer",
      "description": "Maximum number of field names."
    },
    "ValidationFieldNameMinCount": {
      "type": "integer",
      "description": "Minimum number of field names."
    },
    "ValidationFieldNames": {
      "type": "array",
      "description": "Expected field names.",
      "items": {
        "type": "string"
      },
      "minItems": 1
    },
    "FieldValidationType": {
      "type": "string",
      "description": "Type of field validation to run on field.",
      "enum": [
        "equal",
        "null",
        "contains",
        "unique",
        "lessThan",
        "greaterThan",
        "between",
        "in",
        "matches",
        "matchesList",
        "startsWith",
        "endsWith",
        "size",
        "lessThanSize",
        "greaterThanSize",
        "luhnCheck",
        "hasType",
        "hasTypes",
        "distinctInSet",
        "distinctContainsSet",
        "distinctEqual",
        "maxBetween",
        "meanBetween",
        "medianBetween",
        "minBetween",
        "stdDevBetween",
        "sumBetween",
        "lengthBetween",
        "lengthEqual",
        "isDecreasing",
        "isIncreasing",
        "isJsonParsable",
        "matchJsonSchema",
        "matchDateTimeFormat",
        "mostCommonValueInSet",
        "uniqueValuesProportionBetween",
        "quantileValuesBetween"
      ]
    },
    "FieldValidationNegate": {
      "type": "boolean",
      "description": "Negate the validation rule.",
      "default": false
    },
    "FieldValidationStrictly": {
      "type": "boolean",
      "description": "Strictly compare the value. For example, if set to false for lessThan, the comparison will be <=.",
      "default": true
    },
    "FieldValidationAnyValue": {
      "description": "Value to compare against.",
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "number"
        },
        {
          "type": "integer"
        },
        {
          "type": "boolean"
        },
        {
          "type": "object"
        },
        {
          "type": "array"
        }
      ]
    },
    "FieldValidation": {
      "type": "object",
      "properties": {
        "type": {
          "$ref": "#/$defs/FieldValidationType"
        }
      },
      "required": ["type"],
      "unevaluatedProperties": false,
      "allOf": [
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["equal"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "value": {
                "$ref": "#/$defs/FieldValidationAnyValue"
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["value"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["null", "unique", "luhnCheck", "isJsonParsable"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["contains", "startsWith", "endsWith", "hasType"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "value": {
                "type": "string",
                "description": "Value to check if field contains."
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["value"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["matches"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "regex": {
                "type": "string",
                "description": "Regex pattern to match against."
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["regex"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["matchJsonSchema"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "schema": {
                "type": "string",
                "description": "JSON schema to match against."
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["schema"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["matchDateTimeFormat"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "format": {
                "type": "string",
                "description": "DateTime format to match against."
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["format"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["lessThan", "greaterThan"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "value": {
                "$ref": "#/$defs/FieldValidationAnyValue"
              },
              "strictly": {
                "$ref": "#/$defs/FieldValidationStrictly"
              }
            },
            "required": ["value"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": [
                  "maxBetween",
                  "meanBetween",
                  "medianBetween",
                  "minBetween",
                  "stdDevBetween",
                  "sumBetween",
                  "lengthBetween",
                  "uniqueValuesProportionBetween"
                ]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "min": {
                "$ref": "#/$defs/FieldValidationAnyValue"
              },
              "max": {
                "$ref": "#/$defs/FieldValidationAnyValue"
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["min", "max"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["isDecreasing", "isIncreasing"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "strictly": {
                "$ref": "#/$defs/FieldValidationStrictly"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": [
                  "distinctInSet",
                  "distinctContainsSet",
                  "distinctEqual",
                  "mostCommonValueInSet"
                ]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "values": {
                "type": "array",
                "items": {
                  "$ref": "#/$defs/FieldValidationAnyValue"
                }
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["values"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["hasTypes"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "values": {
                "type": "array",
                "description": "List of data types to check for.",
                "items": {
                  "type": "string"
                }
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["values"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["size", "lessThanSize", "greaterThanSize"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "size": {
                "type": "integer",
                "description": "Expected size of field."
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["size"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["matchesList"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "regexes": {
                "type": "array",
                "description": "List of regex patterns to match against.",
                "items": {
                  "type": "string"
                }
              },
              "matchAll": {
                "type": "boolean",
                "description": "Match all regex patterns.",
                "default": true
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["regexes"]
          }
        },
        {
          "if": {
            "properties": {
              "type": {
                "enum": ["quantileValuesBetween"]
              }
            },
            "required": ["type"]
          },
          "then": {
            "properties": {
              "quantileRanges": {
                "type": "object",
                "description": "Quantile ranges to check for.",
                "patternProperties": {
                  "^([0,1]\\.\\d+)$": {
                    "type": "array",
                    "items": {
                      "type": "array",
                      "items": {
                        "type": "number"
                      },
                      "maxItems": 2,
                      "minItems": 2
                    },
                    "maxItems": 1,
                    "minItems": 1
                  }
                }
              },
              "negate": {
                "$ref": "#/$defs/FieldValidationNegate"
              }
            },
            "required": ["quantileRanges"]
          }
        }
      ]
    },
    "FieldValidations": {
      "type": "object",
      "properties": {
        "field": {
          "type": "string",
          "description": "Field name to run validation on."
        },
        "validation": {
          "type": "array",
          "description": "List of validation rules to run on the field.",
          "items": {
            "$ref": "#/$defs/FieldValidation"
          },
          "minItems": 1
        }
      },
      "required": ["field", "validation"],
      "unevaluatedProperties": false
    },
    "BasicValidation": {
      "type": "object",
      "properties": {
        "expr": {
          "$ref": "#/$defs/ValidationExpression"
        },
        "selectExpr": {
          "$ref": "#/$defs/ValidationSelectExpression"
        }
      },
      "anyOf": [
        {
          "$ref": "#/$defs/ValidationAdditionalProperties"
        }
      ],
      "required": ["expr"],
      "unevaluatedProperties": false
    },
    "AggregationValidation": {
      "type": "object",
      "properties": {
        "aggField": {
          "$ref": "#/$defs/ValidationAggregateField"
        },
        "aggType": {
          "$ref": "#/$defs/ValidationAggregateType"
        },
        "groupByFields": {
          "$ref": "#/$defs/ValidationAggregateGroupByFields"
        },
        "aggExpr": {
          "$ref": "#/$defs/ValidationAggregateExpression"
        }
      },
      "anyOf": [
        {
          "$ref": "#/$defs/ValidationAdditionalProperties"
        }
      ],
      "required": ["aggType"],
      "unevaluatedProperties": false
    },
    "UpstreamValidation": {
      "type": "object",
      "properties": {
        "upstreamDataSource": {
          "$ref": "#/$defs/ValidationUpstreamDataSource"
        },
        "joinFields": {
          "$ref": "#/$defs/ValidationUpstreamJoinFields"
        },
        "joinType": {
          "$ref": "#/$defs/ValidationUpstreamJoinType"
        },
        "validations": {
          "$ref": "#/$defs/ValidationUpstreamNestedValidations"
        },
        "upstreamReadOptions": {
          "$ref": "#/$defs/ValidationUpstreamReadOptions"
        }
      },
      "anyOf": [
        {
          "$ref": "#/$defs/ValidationAdditionalProperties"
        }
      ],
      "required": ["upstreamDataSource", "joinFields", "validations"],
      "unevaluatedProperties": false
    },
    "FieldNameValidation": {
      "type": "object",
      "properties": {
        "fieldNameType": {
          "$ref": "#/$defs/ValidationFieldNameType"
        },
        "count": {
          "$ref": "#/$defs/ValidationFieldNameCount"
        },
        "maxCount": {
          "$ref": "#/$defs/ValidationFieldNameMaxCount"
        },
        "minCount": {
          "$ref": "#/$defs/ValidationFieldNameMinCount"
        },
        "names": {
          "$ref": "#/$defs/ValidationFieldNames"
        }
      },
      "anyOf": [
        {
          "$ref": "#/$defs/ValidationAdditionalProperties"
        }
      ],
      "required": ["fieldNameType"],
      "unevaluatedProperties": false
    }
  }
}