Skip to content

Commit

Permalink
Merge branch 'k8s-testnet-create-local-remove' into 'master'
Browse files Browse the repository at this point in the history 
chore(k8s-testnet): remove unused create_local

removing this so that we can remove wait_for_event and downgrade k8s crate

also removed other dead code as a result of removing this function 

See merge request dfinity-lab/public/ic!16645
  • Loading branch information
@Luka-DFINITY
Luka-DFINITY committed Dec 11, 2023
2 parents ea74452 + 1503175 commit 95fe08c
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 226 deletions.
95 changes: 0 additions & 95 deletions rs/tests/k8s/src/tnet.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ static TNET_NAME_LABEL: &str = "tnet.internetcomputer.org/name";
static TNET_TERMINATE_TIME_ANNOTATION: &str = "tnet.internetcomputer.org/terminate-time";

pub struct K8sClient {
pub(crate) client: Client,
pub(crate) api_dv: Api<DynamicObject>,
pub(crate) api_vm: Api<DynamicObject>,
pub(crate) api_vmi: Api<DynamicObject>,
Expand All @@ -83,7 +82,6 @@ impl K8sClient {
let api_vmi = Api::<DynamicObject>::namespaced_with(client.clone(), &TNET_NAMESPACE, &ar);

Ok(Self {
client,
api_dv,
api_vm,
api_vmi,
Expand All @@ -110,7 +108,6 @@ pub struct TNet {
ipv6_net: Option<Ipv6Cidr>,
config_url: Option<String>,
pub(crate) index: Option<u32>,
pub(crate) namespace: String,
pub nns_nodes: Vec<TNode>,
pub app_nodes: Vec<TNode>,
k8s: Option<K8sClient>,
Expand All @@ -122,7 +119,6 @@ impl TNet {
pub fn new(name: &str) -> Result<Self> {
Self {
name: name.to_string(),
namespace: TNET_NAMESPACE.clone(),
..Default::default()
}
.ttl(Duration::days(1))
Expand Down Expand Up @@ -583,97 +579,6 @@ impl TNet {

Ok(self)
}

pub async fn create_local(&mut self) -> Result<&Self> {
self.tnet_owner().await?;
let k8s_client = &self.k8s.as_ref().unwrap();

// generate and upload node config images
if self.init {
generate_config(
&self.version,
self.use_zero_version,
&self.nns_nodes,
&self.app_nodes,
)?;
self.upload_config().await?;
}

// create virtual machines
for node in self.nns_nodes.iter().chain(self.app_nodes.iter()) {
prepare_host_vm(self.k8s.as_ref().unwrap(), node, self).await?;
}

for node in self.nns_nodes.iter().chain(self.app_nodes.iter()) {
create_host_vm(self.k8s.as_ref().unwrap(), node, self).await?;
}

let config_url = format!("{}/init.tar", self.config_url.clone().unwrap());
let nns_ips = self
.nns_nodes
.iter()
.map(|node| node.ipv6_addr.unwrap().to_string())
.collect::<Vec<String>>()
.join(" ");

// initialize nns
// TODO: save init state somehere (host-based pvc)
// initialize nns
if self.init {
create_pod(
&k8s_client.api_pod,
"tnet-operator",
"ubuntu:20.04",
vec![
"/usr/bin/bash",
"-c",
&format!(
r#"
set -eEuo pipefail
if [ -e /mnt/ic-nns-init.complete ]; then
echo NNS already initialized, nothing to do
exit 0
fi
apt update && apt install -y parallel wget iputils-ping libssl1.1="1.1.1f-1ubuntu2"
pushd /mnt
wget {}
tar -xf init.tar
popd
gunzip /mnt/*.gz /mnt/canisters/*.gz || true
chmod u+x /mnt/ic-nns-init
timeout 10m bash -c 'until parallel -u ping -c1 -W1 ::: {} >/dev/null;
do
echo Waiting for NNS nodes to come up...
sleep 5
done'
echo NNS nodes seem to be up...
sleep 30
echo Initiliazing NNS nodes...
/mnt/ic-nns-init --url 'http://[{}]:8080' \
--registry-local-store-dir /mnt/ic_registry_local_store \
--wasm-dir /mnt/canisters --http2-only 2>&1 | tee /mnt/ic-nns-init.log
touch /mnt/ic-nns-init.complete
"#,
config_url, nns_ips, self.nns_nodes[0].ipv6_addr.unwrap()
),
],
vec![
"/usr/bin/bash",
"-c",
"tail -f /dev/null",
],
None,
self.owner_reference(),
)
.await?;
}

Ok(self)
}
}

#[cfg(test)]
Expand Down
132 changes: 1 addition & 131 deletions rs/tests/k8s/src/virtualmachine.rs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
use std::collections::BTreeMap;

use anyhow::{anyhow, Result};
use anyhow::Result;
use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
use kube::api::DynamicObject;
use kube::api::{Patch, PatchParams};
use kube::Api;
use tracing::*;

use crate::event::*;
use crate::pod::get_pods;
use crate::tnet::K8sClient;
use crate::tnet::{TNet, TNode};

Expand Down Expand Up @@ -72,70 +68,6 @@ spec:
name: disk1
"#;

static VM_HOST_DISK_TEMPLATE: &str = r#"
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
name: {name}
tnet.internetcomputer.org/name: {tnet}
spec:
running: true
template:
metadata:
annotations:
"cni.projectcalico.org/ipAddrs": '["{ipv6}"]'
"container.apparmor.security.beta.kubernetes.io/compute": "unconfined"
labels:
kubevirt.io/network: passt
kubevirt.io/vm: {name}
spec:
nodeSelector:
kubernetes.io/hostname: {nodename}
domain:
cpu:
cores: 32
devices:
disks:
- disk:
bus: virtio
name: host-disk-guestos
- disk:
bus: scsi
name: host-disk-config
serial: config
interfaces:
- name: default
passt: {}
ports:
- port: 22
- port: 4100
- port: 2497
- port: 8080
- port: 9090
- port: 9100
firmware:
bootloader:
efi:
secureBoot: false
machine:
type: q35
resources:
requests:
memory: 64Gi
networks:
- name: default
pod: {}
volumes:
- hostDisk:
path: /tnet/{tnet}/{name}-guestos.img
type: Disk
name: host-disk-guestos
- hostDisk:
path: /tnet/{tnet}/{name}-config.img
type: Disk
name: host-disk-config
"#;

static VMI_SETUP_TEMPLATE: &str = r#"
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
Expand Down Expand Up @@ -230,68 +162,6 @@ pub async fn prepare_host_vm(k8s_client: &K8sClient, node: &TNode, tnet: &TNet)
Ok(())
}

pub async fn create_host_vm(k8s_client: &K8sClient, node: &TNode, tnet: &TNet) -> Result<()> {
let vm_name: String = node.name.as_ref().unwrap().to_string();
let vmi_name: String = format!("{}-disk-setup", &node.name.as_ref().unwrap());

wait_for_event(
k8s_client.client.clone(),
&tnet.namespace,
"Signaled Deletion",
"VirtualMachineInstance",
&vmi_name,
600,
)
.await
.unwrap_or_else(|_| {
error!(
"Timeout waiting for VirtualMachineInstance {} to create local image disks",
&vmi_name
);
std::process::exit(124);
});
info!("Preparing disks for virtual machine {} complete", &vm_name);

let labels = Some(BTreeMap::from([(
"kubevirt.io/vm".to_string(),
vm_name.to_string(),
)]));
let pods = get_pods(&k8s_client.api_pod, labels).await?;
if pods.items.is_empty() {
return Err(anyhow!("No setup vmi found!"));
} else if pods.items.len() > 1 {
return Err(anyhow!("Multiple setup vmis found!"));
}
let nodename = pods.items[0]
.metadata
.labels
.as_ref()
.unwrap()
.get("kubevirt.io/nodeName")
.unwrap();

info!("Creating virtual machine {}", &vm_name);
let yaml = VM_HOST_DISK_TEMPLATE
.replace("{nodename}", nodename)
.replace("{name}", &vm_name)
.replace("{tnet}", &TNet::owner_config_map_name(tnet.index.unwrap()))
.replace("{ipv6}", &node.ipv6_addr.as_ref().unwrap().to_string());
let mut data: DynamicObject = serde_yaml::from_str(&yaml)?;
data.metadata.owner_references = vec![tnet.owner_reference()].into();
let response = &k8s_client
.api_vm
.patch(
&vm_name,
&PatchParams::apply("system-driver"),
&Patch::Apply(data),
)
.await?;
debug!("Creating virtual machine response: {:?}", response);
info!("Creating virtual machine {} complete", vm_name);

Ok(())
}

pub async fn create_vm(
api: &Api<DynamicObject>,
name: &str,
Expand Down

0 comments on commit 95fe08c

Please sign in to comment.