Skip to content

Commit

Permalink
Use butane generated static machine config for certs
Browse files Browse the repository at this point in the history 
It is appropriate to generate certificates offline with certutil
and butane, Then import those certficates into worker nodes via
machine config. Hence this commit makes use of such a machine
config to import certficates instead of doing it from a pod.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>
  • Loading branch information
@pperiyasamy
pperiyasamy committed Apr 11, 2024
1 parent 0a75634 commit 70d191c
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 195 deletions.
163 changes: 10 additions & 153 deletions test/extended/networking/ipsec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,158 +30,14 @@ const (
masterIPsecMachineConfigName = "80-ipsec-master-extensions"
workerIPSecMachineConfigName = "80-ipsec-worker-extensions"
nmstateConfigureManifestFile = "nmstate.yaml"
cnoNamespace = "openshift-network-operator"
cnoSAName = "cluster-network-operator"
nsCertMachineConfigFile = "ipsec-nsconfig-machine-config.yaml"
nsCertMachineConfigName = "99-worker-north-south-ipsec-config"
leftNodeIPsecPolicyName = "left-node-ipsec-policy"
rightNodeIPsecPolicyName = "right-node-ipsec-policy"
leftNodeIPsecConfigYaml = "ipsec-left-node.yaml"
rightNodeIPsecConfigYaml = "ipsec-right-node.yaml"
)

var ipsecCertConfigScript = `#!/bin/bash
set -o nounset
set -o errexit
set -o pipefail
nssdb="/var/lib/ipsec/nss"
tmp_dir=/tmp/ipsec-ns
CERTUTIL_NOISE="dsadasdasdasdadasdasdasdasdsadfwerwerjfdksdjfksdlfhjsdk"
mcp_role="worker"
left_ip=$LEFT_IP
right_ip=$RIGHT_IP
if [ -d "$tmp_dir" ]; then
echo "ipsec-ns directory exists. Deleting and recreating..."
rm -r "$tmp_dir"
fi
mkdir "$tmp_dir" && cd "$tmp_dir"
echo "\n-----Create Certs and export to mcp file----\n"
certutil_noise_file="${tmp_dir}/certutil-noise.txt"
echo ${CERTUTIL_NOISE} > ${certutil_noise_file}
create_cert()
{
echo "----Creating CA cert-----"
certutil -v 120 -S -k rsa -n "CA" -s "CN=CA" -v 12 -t "CT,C,C" -x -d ${nssdb} -z ${certutil_noise_file}
sleep 5s
echo "CA cert was created"
}
create_left_user_cert()
{
echo "----Creating left server cert.----"
certutil -v 120 -S -k rsa -c "CA" -n "left_server" -s "CN=left_server" -v 12 -t "u,u,u" -d ${nssdb} --extSAN "ip:${left_ip}" -z ${certutil_noise_file}
sleep 5s
echo "Left server cert was created"
}
create_right_user_cert()
{
echo "-----Creating right server cert.-----"
certutil -v 120 -S -k rsa -c "CA" -n "right_server" -s "CN=right_server" -v 12 -t "u,u,u" -d ${nssdb} --extSAN "ip:${right_ip}" -z ${certutil_noise_file}
sleep 5s
echo "Right server cert was created"
}
export_ca_cert()
{
echo "---Exporting CA cert to p12 and output to pem-----"
pk12util -o ${tmp_dir}/ca.p12 -n CA -d ${nssdb} -W ""
openssl pkcs12 -in ca.p12 -out ca.pem -clcerts -nokeys -passin pass:""
}
export_left_user_cert()
{
echo "----Exporting left user cert to p12-------"
pk12util -o $tmp_dir/left_server.p12 -n left_server -d $nssdb -W ""
}
export_right_user_cert()
{
echo "----Exporting right user cert to p12-------"
pk12util -o $tmp_dir/right_server.p12 -n right_server -d $nssdb -W ""
}
echo "Create certifications and export CA and certs!!"
create_cert
create_left_user_cert
create_right_user_cert
export_ca_cert
export_left_user_cert
export_right_user_cert
chmod 644 $tmp_dir/ca.pem
chmod 644 $tmp_dir/left_server.p12
echo "Create bu file for ipsec configuration on the host!"
cat > $tmp_dir/config.bu <<EOF
variant: openshift
version: 4.15.0
metadata:
name: %s
labels:
machineconfiguration.openshift.io/role: ${mcp_role}
systemd:
units:
- name: ipsec-import.service
enabled: true
contents: |
[Unit]
Description=Import external certs into ipsec NSS
Before=ipsec.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/ipsec-addcert.sh
RemainAfterExit=false
StandardOutput=journal
[Install]
WantedBy=multi-user.target
storage:
files:
- path: /etc/pki/certs/ca.pem
mode: 0400
overwrite: true
contents:
local: ca.pem
- path: /etc/pki/certs/left_server.p12
mode: 0400
overwrite: true
contents:
local: left_server.p12
- path: /etc/pki/certs/right_server.p12
mode: 0400
overwrite: true
contents:
local: right_server.p12
- path: /usr/local/bin/ipsec-addcert.sh
mode: 0740
overwrite: true
contents:
inline: |
#!/bin/bash -e
echo "importing cert to NSS"
certutil -A -n "CA" -t "CT,C,C" -d /var/lib/ipsec/nss/ -i /etc/pki/certs/ca.pem
pk12util -W "" -i /etc/pki/certs/left_server.p12 -d /var/lib/ipsec/nss/
certutil -M -n "left_server" -t "u,u,u" -d /var/lib/ipsec/nss/
pk12util -W "" -i /etc/pki/certs/right_server.p12 -d /var/lib/ipsec/nss/
certutil -M -n "right_server" -t "u,u,u" -d /var/lib/ipsec/nss/
EOF
echo "Creating mcp file..."
butane --files-dir $tmp_dir $tmp_dir/config.bu -o $tmp_dir/config_ipsec_ns.yaml
echo "Importing certs to worker nodes."
kubectl apply -f $tmp_dir/config_ipsec_ns.yaml
echo "IPSEC North-South configuration completed!"
sleep infinity
`

var nodeIPsecConfigManifest = `
kind: NodeNetworkConfigurationPolicy
apiVersion: nmstate.io/v1
Expand Down Expand Up @@ -213,6 +69,9 @@ spec:
type: transport
`

// Expiration date for certificates contained in the nsCertMachineConfigFile.
var certExpirationDate = time.Date(2034, time.April, 10, 0, 0, 0, 0, time.UTC)

// configureIPsec helps to rollout specified IPsecConfig on the cluster.
func configureIPsec(oc *exutil.CLI, ipsecConfig *v1.IPsecConfig) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
Expand Down Expand Up @@ -717,15 +576,13 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
waitForIPsecConfigToComplete(oc, v1.IPsecModeExternal)
checkForGeneveOnlyTraffic(config)

// TODO: replace private image with network-tools after updating it with
// required packages (libreswan, bluto, openssl) to configure the certs.
// It also spins up the pod in openshift-network-operator namespace so that
// pod has ability to rollout a machine config for importing the certs into
// worker nodes.
g.By("configure IPsec certs on the worker nodes")
nsCertMachineConfig, err := launchIPsecCertsConfig(oc, "quay.io/pepalani/ipsec-tools:latest",
config.srcNodeConfig.nodeIP, config.dstNodeConfig.nodeIP, config.controlPlaneNode,
"ipsec-ns-config")
// The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024.
// so proceed with test if system date is before April 10, 2034. Otherwise fail the test.
if !time.Now().Before(certExpirationDate) {
framework.Failf("certficates in the Machine Config are expired, Please consider recreating those certificates")
}
nsCertMachineConfig, err := createIPsecCertsMachineConfig(oc)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(nsCertMachineConfig).NotTo(o.BeNil())
o.Eventually(func() bool {
Expand Down
45 changes: 4 additions & 41 deletions test/extended/networking/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -811,51 +811,14 @@ func isDaemonSetRunning(oc *exutil.CLI, namespace, name string) (bool, error) {
return ready, nil
}

func launchIPsecCertsConfig(oc *exutil.CLI, configImage, leftIP, rightIP, nodeName, generateName string) (*mcfgv1.MachineConfig, error) {
pod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
GenerateName: generateName,
Namespace: cnoNamespace,
},
Spec: v1.PodSpec{
ServiceAccountName: cnoSAName,
Containers: []v1.Container{
{
Name: fmt.Sprintf("%s-container", generateName),
Image: configImage,
Command: []string{
"/bin/bash",
"-c",
fmt.Sprintf(ipsecCertConfigScript, nsCertMachineConfigName),
},
Env: []v1.EnvVar{
{Name: "LEFT_IP", Value: leftIP},
{Name: "RIGHT_IP", Value: rightIP},
},
},
},
NodeName: nodeName,
RestartPolicy: v1.RestartPolicyNever,
HostNetwork: true,
HostPID: true,
},
}
p, err := oc.AdminKubeClient().CoreV1().Pods(cnoNamespace).Create(context.Background(), pod, metav1.CreateOptions{})
func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error) {
ipSecCertsMachineConfig := exutil.FixturePath("testdata", "ipsec", nsCertMachineConfigFile)
err := oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute()
if err != nil {
return nil, err
return nil, fmt.Errorf("error deploying IPsec certs Machine Config: %v", err)
}
var nsCertMachineConfig *mcfgv1.MachineConfig
err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
retrievedPod, err := oc.AdminKubeClient().CoreV1().Pods(cnoNamespace).Get(ctx, p.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if retrievedPod.Status.Phase != corev1.PodRunning {
return false, nil
}
nsCertMachineConfig, err = oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(),
nsCertMachineConfigName, metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
Expand Down

0 comments on commit 70d191c

Please sign in to comment.