Added OSS weaviate setup with same infra as shown in https://redis.io/blog/benchmarking-results-for-vector-databases/

terraform/weaviate-v1.29.1-m6i-2xlarge/Readme.md

@@ -0,0 +1,6 @@
+
+apt update
+apt install docker.io docker-compose -y
+
+git clone https://github.com/redis-performance/vector-db-benchmark
+cd vector-db-benchmark

terraform/weaviate-v1.29.1-m6i-2xlarge/common.tf

@@ -0,0 +1,12 @@
+
+################################################################################
+# This is the bucket holding this specific setup tfstate
+################################################################################
+terraform {
+  backend "s3" {
+    bucket = "performance-cto-group"
+    region = "us-east-1"
+    key = "weaviate-v1.29.1-m6i-2xlarge.tfstate"
+  }
+}
+

terraform/weaviate-v1.29.1-m6i-2xlarge/db-resources_2a.tf

@@ -0,0 +1,63 @@
+
+resource "aws_instance" "server_2a" {
+  count                  = var.server_instance_count
+  ami                    = var.instance_ami
+  instance_type          = var.server_instance_type
+  subnet_id              = data.terraform_remote_state.shared_resources.outputs.subnet_public_id
+  vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name               = var.key_name
+  availability_zone      = "us-east-2a"
+  placement_group        = data.terraform_remote_state.shared_resources.outputs.perf_cto_pg_name
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+  }
+
+  volume_tags = {
+    Environment    = "${var.environment}"
+    Project        = "${var.environment}"
+    Name           = "ebs_block_device-${var.setup_name}-DB-us-east-2a-${count.index + 1}"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+  }
+
+  tags = {
+    Environment    = "${var.environment}"
+    Project        = "${var.environment}"
+    Name           = "${var.setup_name}-DB-us-east-2a-${count.index + 1}"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+  }
+
+  ################################################################################
+  # Deployment related
+  ################################################################################
+
+  ################################################################################
+  # Install docker
+  ################################################################################
+  provisioner "remote-exec" {
+    script = "./install_docker.sh"
+    connection {
+      host        = self.public_ip # The `self` variable is like `this` in many programming languages
+      type        = "ssh"          # in this case, `self` is the resource (the server).
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      #need to increase timeout to larger then 5m for metal instances
+      timeout = "5m"
+      agent   = "false"
+    }
+  }
+
+}

terraform/weaviate-v1.29.1-m6i-2xlarge/install_docker.sh

@@ -0,0 +1,20 @@
+export DEBIAN_FRONTEND=noninteractive
+
+sudo apt-get update -y
+sudo apt-get install ca-certificates curl -y
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update -y
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+
+# check
+echo "Printing docker info"
+docker --version

terraform/weaviate-v1.29.1-m6i-2xlarge/output.tf

@@ -0,0 +1,23 @@
+output "server_public_ip" {
+  value = aws_instance.server_2a[*].public_ip
+}
+
+output "server_private_ip" {
+  value = aws_instance.server_2a[*].private_ip
+}
+
+output "server_instance_type" {
+  value = var.server_instance_type
+}
+
+output "search_threads" {
+  value = var.search_thread
+}
+
+output "setup_name" {
+  value = var.setup_name
+}
+
+output "server_instance_count" {
+  value = var.server_instance_count
+}

terraform/weaviate-v1.29.1-m6i-2xlarge/shared_resources.tf

@@ -0,0 +1,17 @@
+# provider
+provider "aws" {
+  region = var.region
+}
+
+################################################################################
+# This is the shared resources bucket key -- you will need it across environments like security rules,etc...
+# !! do not change this !!
+################################################################################
+data "terraform_remote_state" "shared_resources" {
+  backend = "s3"
+  config = {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/shared_resources.tfstate"
+    region = "us-east-1"
+  }
+}

terraform/weaviate-v1.29.1-m6i-2xlarge/variables.tf

@@ -0,0 +1,165 @@
+################################################################################
+# Variables used for deployment tag
+################################################################################
+
+variable "search_thread" {
+  description = "setup name"
+  default     = "6"
+}
+
+variable "setup_name" {
+  description = "setup name"
+  default     = "weaviate-v1.29.1-m6i-2xlarge"
+}
+
+variable "github_actor" {
+  description = "The name of the person or app that initiated the deployment."
+  default     = "N/A"
+}
+
+
+variable "github_repo" {
+  description = "	The owner and repository name. For example, testing-infrastructure."
+  default     = "N/A"
+}
+
+variable "triggering_env" {
+  description = "	The triggering environment. For example circleci."
+  default     = "N/A"
+}
+
+variable "environment" {
+  description = "	The cost tag."
+  default     = "VecSim-Competitive"
+}
+
+variable "github_org" {
+  description = "	The owner name. For example, RedisModules."
+  default     = "N/A"
+}
+
+variable "github_sha" {
+  description = "The commit SHA that triggered the deployment."
+  default     = "N/A"
+}
+
+variable "timeout_secs" {
+  description = "The maximum time to wait prior destroying the VM via the watchdog."
+  default     = "3600"
+}
+
+
+
+################################################################################
+# Access keys
+################################################################################
+variable "private_key" {
+  description = "private key"
+  default     = "/tmp/benchmarks.redislabs.pem"
+}
+
+variable "key_name" {
+  description = "key name"
+  default     = "perf-cto-us-east-2"
+}
+
+variable "region" {
+  default = "us-east-2"
+}
+
+# (Ubuntu 20.04)
+# us-east-2	Focal Fossa	20.04 LTS	amd64	hvm:ebs-ssd	20240205	ami-07b469810a61205a8	hvm
+variable "instance_ami" {
+  description = "AMI for aws EC2 instance - us-east-2 Ubuntu 20.04 - amd64"
+  default     = "ami-07b469810a61205a8"
+}
+
+variable "instance_device_name" {
+  description = "EC2 instance device name"
+  default     = "/dev/sda1"
+}
+
+variable "redis_module" {
+  description = "redis_module"
+  default     = "N/A"
+}
+
+variable "instance_volume_size" {
+  description = "EC2 instance volume_size"
+  default     = "256"
+}
+
+variable "instance_volume_type" {
+  description = "EC2 instance volume_type"
+  default     = "gp3"
+}
+
+
+variable "instance_volume_iops" {
+  description = "EC2 instance volume_iops"
+  default     = "3000"
+}
+
+variable "client_instance_volume_size" {
+  description = "EC2 instance volume_size"
+  default     = "256"
+}
+
+variable "client_instance_volume_type" {
+  description = "EC2 instance volume_type"
+  default     = "gp3"
+}
+
+
+variable "instance_volume_encrypted" {
+  description = "EC2 instance instance_volume_encrypted"
+  default     = "false"
+}
+
+variable "instance_root_block_device_encrypted" {
+  description = "EC2 instance instance_root_block_device_encrypted"
+  default     = "false"
+}
+
+variable "instance_cpu_threads_per_core" {
+  description = "CPU threads per core for aws EC2 instance"
+  default     = 1
+}
+
+variable "instance_cpu_threads_per_core_hyperthreading" {
+  description = "CPU threads per core when hyperthreading is enabled for aws EC2 instance"
+  default     = 2
+}
+
+variable "instance_network_interface_plus_count" {
+  description = "number of additional network interfaces to add to aws EC2 instance"
+  default     = 0
+}
+
+variable "os" {
+  description = "os"
+  default     = "ubuntu20.04"
+}
+
+variable "ssh_user" {
+  description = "ssh_user"
+  default     = "ubuntu"
+}
+
+################################################################################
+# Specific DB machine variables
+################################################################################
+variable "server_instance_type" {
+  description = "type for aws EC2 instance"
+  default     = "m6i.2xlarge"
+}
+
+variable "server_instance_count" {
+  description = "count of aws EC2 instances"
+  default     = 1
+}
+
+variable "server_instance_cpu_core_count" {
+  description = "CPU core count for aws EC2 instance"
+  default     = 4
+}