Gather Kuryr specific data

This commit ensures that all kuryr related data is gathered.
openshift · Mar 11, 2020 · e17d77d · e17d77d
1 parent 3cac0b5
commit e17d77d
Showing 1 changed file with 126 additions and 74 deletions.
diff --git a/collection-scripts/gather_network_logs b/collection-scripts/gather_network_logs
@@ -1,91 +1,143 @@
 #!/bin/bash
 BASE_COLLECTION_PATH="/must-gather"
-NETWORK_LOG_PATH="${BASE_COLLECTION_PATH}/network_logs/"
+NETWORK_LOG_PATH="${BASE_COLLECTION_PATH}/network_logs"
 
 mkdir -p ${NETWORK_LOG_PATH}/
 
+function gather_openshiftsdn_nodes_data {
+  for NODE in $NODES; do
+      SDN_POD=$(oc -n openshift-sdn get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=sdn)
+      oc -n openshift-sdn exec $SDN_POD -- bash -c "iptables-save -c" > ${NETWORK_LOG_PATH}/${NODE}_iptables &
+      PIDS+=($!)
+      oc -n openshift-sdn exec $SDN_POD -- bash -c "ovs-vsctl show" > ${NETWORK_LOG_PATH}/${NODE}_ovs_dump &
+      PIDS+=($!)
+
+      OVS_POD=$(oc -n openshift-sdn get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=ovs)
+      oc -n openshift-sdn exec $OVS_POD -- bash -c "cat /var/log/openvswitch/ovs-vswitchd.log" > ${NETWORK_LOG_PATH}/${OVS_POD}_vswitchd_log &
+      PIDS+=($!)
+      oc -n openshift-sdn exec $OVS_POD -- bash -c "cat /var/log/openvswitch/ovsdb-server.log" > ${NETWORK_LOG_PATH}/${OVS_POD}_ovsdb_log &
+      PIDS+=($!)
+  done
+}
+
+function gather_kuryr_nodes_data {
+  for NODE in $NODES; do
+      CNI_POD=$(oc -n openshift-kuryr get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=kuryr-cni)
+      oc exec -n openshift-kuryr $CNI_POD -- bash -c 'for pid in $(find /host_proc/[1-9]*/ns/net | cut -d/ -f3); do\
+          echo "Namespace: $pid";\
+          nsenter --net=/host_proc/$pid/ns/net ip -d addr;\
+          done' > ${NETWORK_LOG_PATH}/${CNI_POD}_interfaces & PIDS+=($!)
+  done
+}
+
+function gather_ovn_kubernetes_nodes_data {
+  for NODE in $NODES; do
+      OVNKUBE_NODE_POD=$(oc -n openshift-ovn-kubernetes get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=ovnkube-node)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "iptables-save -c" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_iptables &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip addr" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_addr &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip route" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_route &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip -6 route" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_6_route &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c \
+      "ovs-ofctl dump-ports-desc br-int" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_ports_br_int &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ovs-ofctl dump-flows br-int" > \
+      ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_flows_br_int &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c \
+      "ovs-ofctl dump-ports-desc br-local" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_ports_br_local &
+      PIDS+=($!)
+      oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ovs-ofctl dump-flows br-local" > \
+      ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_flows_br_local &
+      PIDS+=($!)
+  done
+}
+
+function gather_ovn_kubernetes_master_data {
+  OVNKUBE_MASTER_IPS=($(oc -n openshift-ovn-kubernetes get pods -l app=ovnkube-master -o=jsonpath='{.items[*].status.podIP}'))
+  OVNKUBE_MASTER_POD=$(oc -n openshift-ovn-kubernetes get pods --no-headers -o custom-columns=':metadata.name' -l app=ovnkube-master | tail -1)
+
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt show" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_show &
+  PIDS+=($!)
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Logical_Switch_Port" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_lsp &
+  PIDS+=($!)
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Load_Balancer" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_lb &
+  PIDS+=($!)
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Port_Group" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_pg &
+  PIDS+=($!)
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list ACL" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_acl &
+  PIDS+=($!)
+  oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
+  "ovn-sbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9642,ssl:${OVNKUBE_MASTER_IPS[1]}:9642,ssl:${OVNKUBE_MASTER_IPS[2]}:9642 \
+  -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt show" > \
+  ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_sbctl_show &
+  PIDS+=($!)
+}
+
+function gather_kuryr_data {
+  oc get pods -A -o wide --show-labels > ${NETWORK_LOG_PATH}/get_pods & PIDS+=($!)
+  openstack port list > ${NETWORK_LOG_PATH}/openstack_port_list & PIDS+=($!)
+
+  oc get kuryrnets -A > ${NETWORK_LOG_PATH}/get_kuryrnets & PIDS+=($!)
+  openstack network list > ${NETWORK_LOG_PATH}/openstack_network_list & PIDS+=($!)
+  openstack subnet list > ${NETWORK_LOG_PATH}/openstack_subnet_list & PIDS+=($!)
+
+  oc get svc -A > ${NETWORK_LOG_PATH}/get_svc & PIDS+=($!)
+  openstack loadbalancer list > ${NETWORK_LOG_PATH}/openstack_loadbalancer_list & PIDS+=($!)
+
+  ROUTERS="$(openstack router list -c ID -f value)"
+  for ROUTER in $ROUTERS; do
+    openstack router show $ROUTER
+  done > ${NETWORK_LOG_PATH}/openstack_router_list
+
+  NETWORK_TRUNKS="$(openstack network trunk list -c ID -f value)"
+  for NETWORK_TRUNK in $NETWORK_TRUNKS; do
+    openstack network trunk show $NETWORK_TRUNK & PIDS+=($!)
+  done > ${NETWORK_LOG_PATH}/openstack_network_trunk_list
+
+  SECURITY_GROUPS="${@:-$(openstack security group list -c ID -f value)}"
+  for SECURITY_GROUP in $SECURITY_GROUPS; do
+    openstack security group show $SECURITY_GROUP & PIDS+=($!)
+  done > ${NETWORK_LOG_PATH}/openstack_security_group_list
+  openstack quota show > ${NETWORK_LOG_PATH}/openstack_quota_show
+}
+
+
 if [ $# -eq 0 ]; then
     echo "WARNING: Collecting network logs on ALL nodes in your cluster. This could take a large amount of time." >&2
 fi
 
 PIDS=()
-
+NODES="${@:-$(oc get nodes --no-headers -o custom-columns=':metadata.name')}"
 NETWORK_TYPE=$(oc get network.config.openshift.io -o=jsonpath='{.items[0].spec.networkType}' | tr '[:upper:]' '[:lower:]')
 if [ "$NETWORK_TYPE" == "openshiftsdn" ]; then
-    NODES="${@:-$(oc get nodes --no-headers -o custom-columns=':metadata.name')}"
-    for NODE in $NODES; do
-        SDN_POD=$(oc -n openshift-sdn get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=sdn)
-        oc -n openshift-sdn exec $SDN_POD -- bash -c "iptables-save -c" > ${NETWORK_LOG_PATH}/${NODE}_iptables &
-        PIDS+=($!)
-        oc -n openshift-sdn exec $SDN_POD -- bash -c "ovs-vsctl show" > ${NETWORK_LOG_PATH}/${NODE}_ovs_dump &
-        PIDS+=($!)
-
-        OVS_POD=$(oc -n openshift-sdn get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=ovs)
-        oc -n openshift-sdn exec $OVS_POD -- bash -c "cat /var/log/openvswitch/ovs-vswitchd.log" > ${NETWORK_LOG_PATH}/${OVS_POD}_vswitchd_log &
-        PIDS+=($!)
-        oc -n openshift-sdn exec $OVS_POD -- bash -c "cat /var/log/openvswitch/ovsdb-server.log" > ${NETWORK_LOG_PATH}/${OVS_POD}_ovsdb_log &
-        PIDS+=($!)
-    done
+    gather_openshiftsdn_nodes_data
+elif [ "$NETWORK_TYPE" == "kuryr" ]; then
+    gather_kuryr_nodes_data
+    gather_kuryr_data
 else
-    NODES="${@:-$(oc get nodes --no-headers -o custom-columns=':metadata.name')}"
-    for NODE in $NODES; do
-        OVNKUBE_NODE_POD=$(oc -n openshift-ovn-kubernetes get pods --no-headers -o custom-columns=":metadata.name" --field-selector spec.nodeName=$NODE -l app=ovnkube-node)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "iptables-save -c" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_iptables &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip addr" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_addr &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip route" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_route &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ip -6 route" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ip_6_route &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c \
-        "ovs-ofctl dump-ports-desc br-int" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_ports_br_int &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ovs-ofctl dump-flows br-int" > \
-        ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_flows_br_int &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c \
-        "ovs-ofctl dump-ports-desc br-local" > ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_ports_br_local &
-        PIDS+=($!)
-        oc -n openshift-ovn-kubernetes exec -c ovnkube-node $OVNKUBE_NODE_POD -- bash -c "ovs-ofctl dump-flows br-local" > \
-        ${NETWORK_LOG_PATH}/${OVNKUBE_NODE_POD}_ovs_ofctl_dump_flows_br_local &
-        PIDS+=($!)
-    done
-
-    OVNKUBE_MASTER_IPS=($(oc -n openshift-ovn-kubernetes get pods -l app=ovnkube-master -o=jsonpath='{.items[*].status.podIP}'))
-    OVNKUBE_MASTER_POD=$(oc -n openshift-ovn-kubernetes get pods --no-headers -o custom-columns=':metadata.name' -l app=ovnkube-master | tail -1)
-
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt show" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_show &
-    PIDS+=($!)
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Logical_Switch_Port" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_lsp &
-    PIDS+=($!)
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Load_Balancer" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_lb &
-    PIDS+=($!)
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list Port_Group" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_pg &
-    PIDS+=($!)
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-nbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9641,ssl:${OVNKUBE_MASTER_IPS[1]}:9641,ssl:${OVNKUBE_MASTER_IPS[2]}:9641 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt list ACL" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_nbctl_list_acl &
-    PIDS+=($!)
-    oc -n openshift-ovn-kubernetes exec -c ovnkube-master $OVNKUBE_MASTER_POD -- bash -c \
-    "ovn-sbctl --db=ssl:${OVNKUBE_MASTER_IPS[0]}:9642,ssl:${OVNKUBE_MASTER_IPS[1]}:9642,ssl:${OVNKUBE_MASTER_IPS[2]}:9642 \
-    -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt show" > \
-    ${NETWORK_LOG_PATH}/${OVNKUBE_MASTER_POD}_ovn_sbctl_show &
-    PIDS+=($!)
+    gather_ovn_kubernetes_nodes_data
+    gather_ovn_kubernetes_master_data
 fi
+
 echo "INFO: Waiting for node network log collection to complete ..."
 wait ${PIDS[@]}
 echo "INFO: Node network log collection to complete."