application.conf

flags {
    enableCount = false
    enableCount = ${?ENABLE_COUNT}
    enableGenerateData = true
    enableGenerateData = ${?ENABLE_GENERATE_DATA}
    enableGeneratePlanAndTasks = false
    enableGeneratePlanAndTasks = ${?ENABLE_GENERATE_PLAN_AND_TASKS}
    enableRecordTracking = false
    enableRecordTracking = ${?ENABLE_RECORD_TRACKING}
    enableDeleteGeneratedRecords = false
    enableDeleteGeneratedRecords = ${?ENABLE_DELETE_GENERATED_RECORDS}
    enableSinkMetadata = true
    enableSinkMetadata = ${?ENABLED_SINK_METADATA}
    enableSaveReports = true
    enableSaveReports = ${?ENABLED_SAVE_REPORTS}
    enableValidation = false
    enableValidation = ${?ENABLED_VALIDATION}
}

folders {
    generatedPlanAndTaskFolderPath = "/opt/app/custom/generated"
    generatedPlanAndTaskFolderPath = ${?GENERATED_PLAN_AND_TASK_FOLDER_PATH}
    planFilePath = "/opt/app/custom/plan/postgres-plan.yaml"
    planFilePath = ${?PLAN_FILE_PATH}
    taskFolderPath = "/opt/app/custom/task"
    taskFolderPath = ${?TASK_FOLDER_PATH}
    recordTrackingFolderPath = "/opt/app/custom/recordTracking"
    recordTrackingFolderPath = ${?RECORD_TRACKING_FOLDER_PATH}
    generatedReportsFolderPath = "/opt/app/custom/report"
    generatedReportsFolderPath = ${?GENERATED_REPORTS_FOLDER_PATH}
}

metadata {
    numRecordsFromDataSource = 10000
    numRecordsForAnalysis = 10000
    oneOfDistinctCountVsCountThreshold = 0.1
}

generation {
    numRecordsPerBatch = 1000000
}

runtime {
    master = "local[*]"
    master = ${?DATA_CATERER_MASTER}
    config {
        "spark.driver.memory" = "2g"
        "spark.executor.memory" = "2g"
        "spark.sql.cbo.enabled" = "true"
        "spark.sql.adaptive.enabled" = "true"
        "spark.sql.cbo.planStats.enabled" = "true"
        "spark.sql.legacy.allowUntypedScalaUDF" = "true"
        "spark.sql.statistics.histogram.enabled" = "true"
        "spark.sql.shuffle.partitions" = "4"
        "spark.sql.catalog.postgres" = ""
        "spark.sql.catalog.cassandra" = "com.datastax.spark.connector.datasource.CassandraCatalog"
        "spark.hadoop.fs.s3a.directory.marker.retention" = "keep"
        "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled" = "true"
    }
}

json {
    json {
    }
}

csv {
    csv {
    }
}

parquet {
    parquet {
    }
}

jdbc {
    postgresCustomer {
        url = "jdbc:postgresql://postgresserver:5432/customer"
        url = ${?POSTGRES_URL}
        user = "postgres"
        user = ${?POSTGRES_USER}
        password = "postgres"
        password = ${?POSTGRES_PASSWORD}
        driver = "org.postgresql.Driver"
    }
    mysql {
        url = "jdbc:mysql://mysqlserver:3306/customer"
        url = ${?MYSQL_URL}
        user = "root"
        user = ${?MYSQL_USERNAME}
        password = "root"
        password = ${?MYSQL_PASSWORD}
        driver = "com.mysql.cj.jdbc.Driver"
    }
}


org.apache.spark.sql.cassandra {
    cassandra {
        spark.cassandra.connection.host = "cassandraserver"
        spark.cassandra.connection.host = ${?CASSANDRA_HOST}
        spark.cassandra.connection.port = "9042"
        spark.cassandra.connection.port = ${?CASSANDRA_PORT}
        spark.cassandra.auth.username = "cassandra"
        spark.cassandra.auth.username = ${?CASSANDRA_USERNAME}
        spark.cassandra.auth.password = "cassandra"
        spark.cassandra.auth.password = ${?CASSANDRA_PASSWORD}
    }
}

http {
    httpbin {
    }
}

jms {
    solace {
        initialContextFactory = "com.solacesystems.jndi.SolJNDIInitialContextFactory"
        initialContextFactory = ${?SOLACE_INITIAL_CONTEXT_FACTORY}
        connectionFactory = "/jms/cf/default"
        connectionFactory = ${?SOLACE_CONNECTION_FACTORY}
        url = "smf://solaceserver:55555"
        url = ${?SOLACE_URL}
        user = "admin"
        user = ${?SOLACE_USER}
        password = "admin"
        password = ${?SOLACE_PASSWORD}
        vpnName = "default"
        vpnName = ${?SOLACE_VPN}
    }
}

kafka {
    kafkaAccount {
        kafka.bootstrap.servers = "kafkaserver:29092"
        kafka.bootstrap.servers = ${?KAFKA_BOOTSTRAP_SERVERS}
    }
}