Merge branch 'release/1.14.x' into development

# Conflicts:
#	website/content/api-docs/system/audit.mdx

changelog/24280.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+command/server: display logs on startup immediately if disable-gated-logs flag is set
+```

command/agent.go

@@ -223,6 +223,11 @@ func (c *AgentCommand) Run(args []string) int {
 		InferLevelsWithTimestamp: true,
 	})
 
+	// release log gate if the disable-gated-logs flag is set
+	if c.logFlags.flagDisableGatedLogs {
+		c.logGate.Flush()
+	}
+
 	infoKeys := make([]string, 0, 10)
 	info := make(map[string]string)
 	info["log level"] = config.LogLevel

command/commands.go

@@ -148,6 +148,8 @@ const (
 	flagNameDisableRedirects = "disable-redirects"
 	// flagNameCombineLogs is used to specify whether log output should be combined and sent to stdout
 	flagNameCombineLogs = "combine-logs"
+	// flagDisableGatedLogs is used to disable gated logs and immediately show the vault logs as they become available
+	flagDisableGatedLogs = "disable-gated-logs"
 	// flagNameLogFile is used to specify the path to the log file that Vault should use for logging
 	flagNameLogFile = "log-file"
 	// flagNameLogRotateBytes is the flag used to specify the number of bytes a log file should be before it is rotated.

command/log_flags.go

@@ -15,6 +15,7 @@ import (
 // logFlags are the 'log' related flags that can be shared across commands.
 type logFlags struct {
 	flagCombineLogs       bool
+	flagDisableGatedLogs  bool
 	flagLogLevel          string
 	flagLogFormat         string
 	flagLogFile           string
@@ -41,6 +42,13 @@ func (f *FlagSet) addLogFlags(l *logFlags) {
 		Hidden:  true,
 	})
 
+	f.BoolVar(&BoolVar{
+		Name:    flagDisableGatedLogs,
+		Target:  &l.flagDisableGatedLogs,
+		Default: false,
+		Hidden:  true,
+	})
+
 	f.StringVar(&StringVar{
 		Name:       flagNameLogLevel,
 		Target:     &l.flagLogLevel,

command/proxy.go

@@ -210,6 +210,11 @@ func (c *ProxyCommand) Run(args []string) int {
 	}
 	c.logger = l
 
+	// release log gate if the disable-gated-logs flag is set
+	if c.logFlags.flagDisableGatedLogs {
+		c.logGate.Flush()
+	}
+
 	infoKeys := make([]string, 0, 10)
 	info := make(map[string]string)
 	info["log level"] = config.LogLevel

command/server.go

@@ -1094,6 +1094,11 @@ func (c *ServerCommand) Run(args []string) int {
 	c.logger = l
 	c.allLoggers = append(c.allLoggers, l)
 
+	// flush logs right away if the server is started with the disable-gated-logs flag
+	if c.logFlags.flagDisableGatedLogs {
+		c.flushLog()
+	}
+
 	// reporting Errors found in the config
 	for _, cErr := range configErrors {
 		c.logger.Warn(cErr.String())

enos/enos-globals.hcl

@@ -2,33 +2,44 @@
 # SPDX-License-Identifier: MPL-2.0
 
 globals {
-  backend_tag_key = "VaultStorage"
+  archs            = ["amd64", "arm64"]
+  artifact_sources = ["local", "crt", "artifactory"]
+  artifact_types   = ["bundle", "package"]
+  backends         = ["consul", "raft"]
+  backend_tag_key  = "VaultStorage"
   build_tags = {
     "ce"               = ["ui"]
     "ent"              = ["ui", "enterprise", "ent"]
     "ent.fips1402"     = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.fips1402"]
     "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
     "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
   }
+  consul_versions = ["1.14.11", "1.15.7", "1.16.3", "1.17.0"]
+  distros         = ["ubuntu", "rhel"]
   distro_version = {
     "rhel"   = var.rhel_distro_version
     "ubuntu" = var.ubuntu_distro_version
   }
+  editions = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
   packages = ["jq"]
   distro_packages = {
     ubuntu = ["netcat"]
     rhel   = ["nc"]
   }
   sample_attributes = {
-    # NOTE(9/28/23): Temporarily use us-east-2 due to another networking in us-east-1
-    # aws_region = ["us-east-1", "us-west-2"]
-    aws_region = ["us-east-2", "us-west-2"]
+    aws_region = ["us-east-1", "us-west-2"]
   }
+  seals = ["awskms", "pkcs11", "shamir"]
   tags = merge({
     "Project Name" : var.project_name
     "Project" : "Enos",
     "Environment" : "ci"
   }, var.tags)
+  // NOTE: when backporting, make sure that our initial versions are less than that
+  // release branch's version. Also beware if adding versions below 1.11.x. Some scenarios
+  // that use this global might not work as expected with earlier versions. Below 1.8.x is
+  // not supported in any way.
+  upgrade_initial_versions = ["1.11.12", "1.12.11", "1.13.12", "1.14.8"]
   vault_install_dir_packages = {
     rhel   = "/bin"
     ubuntu = "/usr/bin"

enos/enos-modules.hcl

@@ -49,6 +49,10 @@ module "generate_secondary_token" {
   vault_install_dir = var.vault_install_dir
 }
 
+module "install_packages" {
+  source = "./modules/install_packages"
+}
+
 module "read_license" {
   source = "./modules/read_license"
 }
@@ -57,16 +61,25 @@ module "replication_data" {
   source = "./modules/replication_data"
 }
 
-module "seal_key_awskms" {
-  source = "./modules/seal_key_awskms"
+module "seal_awskms" {
+  source = "./modules/seal_awskms"
 
-  common_tags = var.tags
+  cluster_ssh_keypair = var.aws_ssh_keypair_name
+  common_tags         = var.tags
 }
 
-module "seal_key_shamir" {
-  source = "./modules/seal_key_shamir"
+module "seal_shamir" {
+  source = "./modules/seal_shamir"
 
-  common_tags = var.tags
+  cluster_ssh_keypair = var.aws_ssh_keypair_name
+  common_tags         = var.tags
+}
+
+module "seal_pkcs11" {
+  source = "./modules/seal_pkcs11"
+
+  cluster_ssh_keypair = var.aws_ssh_keypair_name
+  common_tags         = var.tags
 }
 
 module "shutdown_node" {
@@ -213,6 +226,13 @@ module "vault_verify_undo_logs" {
   vault_instance_count = var.vault_instance_count
 }
 
+module "vault_verify_default_lcq" {
+  source = "./modules/vault_verify_default_lcq"
+
+  vault_autopilot_default_max_leases = "300000"
+  vault_instance_count               = var.vault_instance_count
+}
+
 module "vault_verify_replication" {
   source = "./modules/vault_verify_replication"
 
@@ -269,20 +289,17 @@ module "vault_verify_write_data" {
 module "vault_wait_for_leader" {
   source = "./modules/vault_wait_for_leader"
 
-  vault_install_dir    = var.vault_install_dir
-  vault_instance_count = var.vault_instance_count
+  vault_install_dir = var.vault_install_dir
 }
 
 module "vault_wait_for_seal_rewrap" {
   source = "./modules/vault_wait_for_seal_rewrap"
 
-  vault_install_dir    = var.vault_install_dir
-  vault_instance_count = var.vault_instance_count
+  vault_install_dir = var.vault_install_dir
 }
 
 module "verify_seal_type" {
   source = "./modules/verify_seal_type"
 
-  vault_install_dir    = var.vault_install_dir
-  vault_instance_count = var.vault_instance_count
+  vault_install_dir = var.vault_install_dir
 }

enos/enos-scenario-agent.hcl

@@ -3,14 +3,14 @@
 
 scenario "agent" {
   matrix {
-    arch            = ["amd64", "arm64"]
-    artifact_source = ["local", "crt", "artifactory"]
-    artifact_type   = ["bundle", "package"]
-    backend         = ["consul", "raft"]
-    consul_version  = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"]
-    distro          = ["ubuntu", "rhel"]
-    edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
-    seal            = ["awskms", "shamir"]
+    arch            = global.archs
+    artifact_source = global.artifact_sources
+    artifact_type   = global.artifact_types
+    backend         = global.backends
+    consul_version  = global.consul_versions
+    distro          = global.distros
+    edition         = global.editions
+    seal            = global.seals
     seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
@@ -24,6 +24,12 @@ scenario "agent" {
       arch    = ["arm64"]
       edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     }
+
+    # PKCS#11 can only be used on ent.hsm and ent.hsm.fips1402.
+    exclude {
+      seal    = ["pkcs11"]
+      edition = ["ce", "ent", "ent.fips1402"]
+    }
   }
 
   terraform_cli = terraform_cli.default
@@ -82,15 +88,6 @@ scenario "agent" {
     }
   }
 
-  step "create_seal_key" {
-    module = "seal_key_${matrix.seal}"
-
-    variables {
-      cluster_id  = step.create_vpc.cluster_id
-      common_tags = global.tags
-    }
-  }
-
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -111,6 +108,20 @@ scenario "agent" {
     }
   }
 
+  step "create_seal_key" {
+    module     = "seal_${matrix.seal}"
+    depends_on = [step.create_vpc]
+
+    providers = {
+      enos = provider.enos.ubuntu
+    }
+
+    variables {
+      cluster_id  = step.create_vpc.id
+      common_tags = global.tags
+    }
+  }
+
   step "create_vault_cluster_targets" {
     module     = module.target_ec2_instances
     depends_on = [step.create_vpc]
@@ -195,8 +206,8 @@ scenario "agent" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_attributes      = step.create_seal_key.attributes
       seal_ha_beta         = matrix.seal_ha_beta
-      seal_key_name        = step.create_seal_key.resource_name
       seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
@@ -440,9 +451,9 @@ scenario "agent" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
-  output "seal_key_name" {
-    description = "The name of the cluster seal key"
-    value       = step.create_seal_key.resource_name
+  output "seal_attributes" {
+    description = "The Vault cluster seal attributes"
+    value       = step.create_seal_key.attributes
   }
 
   output "unseal_keys_b64" {