Refactor steps (#55)

* Refactoring step1-2, 2-2 and each file path

* Remove unused api of netomox-exp/model-conductor

* remove pni_addlink external-as-scripts (automatically generated data)

* Move bgp operation from step2-1 to 2-2

* add before_flowdata for pni_te

* preferred flag for external-as topology script

* refactor generate.py

* Fix netomox and toolbox version (ruby3.0 compatible)

* remove restart_iperf.sh

* Update env

---------

Co-authored-by: mddo <mddo>

.env

@@ -14,8 +14,8 @@ BATFISH_IMAGE_TAG=v0.3.0
 BATFISH_WRAPPER_IMAGE_TAG=v1.1.1
 BGP_POLICY_PARSER_IMAGE_TAG=v0.5.0
 FISH_TRACER_IMAGE_TAG=v1.0.0
-MODEL_CONDUCTOR_IMAGE_TAG=v1.7.0
-NETOMOX_EXP_IMAGE_TAG=v1.9.0
+MODEL_CONDUCTOR_IMAGE_TAG=v1.8.0
+NETOMOX_EXP_IMAGE_TAG=v1.10.0
 NETOVIZ_IMAGE_TAG=v0.7.0
 
 # container port number

demo/Gemfile

@@ -3,5 +3,7 @@
 source 'https://rubygems.org'
 
 source "https://rubygems.pkg.github.com/ool-mddo" do
-  gem "mddo-toolbox-cli", ">= 1.2.0"
+  gem "mddo-toolbox-cli", ">= 1.2.1"
+  gem "netomox", ">= 0.9.1"
 end
+

demo/Gemfile.lock

@@ -1,12 +1,12 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    excon (0.102.0)
+    excon (0.110.0)
     hashdiff (1.0.1)
-    highline (2.1.0)
+    highline (3.0.1)
     httpclient (2.8.3)
-    json (2.6.3)
-    jsonpath (1.1.3)
+    json (2.7.2)
+    jsonpath (1.1.5)
       multi_json
     multi_json (1.15.0)
     neography (1.8.0)
@@ -18,24 +18,24 @@ GEM
       rubyzip (>= 1.0.0)
     os (1.1.4)
     power_assert (2.0.3)
-    rake (13.0.6)
+    rake (13.2.1)
     rexml (3.2.6)
     rubyzip (2.3.2)
     termcolor (1.2.2)
       highline (>= 1.5.0)
-    test-unit (3.6.1)
+    test-unit (3.6.2)
       power_assert
     thor (1.2.2)
 
 GEM
   remote: https://rubygems.pkg.github.com/ool-mddo/
   specs:
-    mddo-toolbox-cli (1.2.0)
+    mddo-toolbox-cli (1.2.1)
       httpclient (~> 2.8)
-      netomox (>= 0.5.0)
+      netomox (>= 0.9.1)
       test-unit (~> 3.5)
       thor (~> 1.2)
-    netomox (0.5.0)
+    netomox (0.9.1)
       hashdiff (~> 1.0.1)
       jsonpath (~> 1.1.2)
       neography (~> 1.8.0)
@@ -47,7 +47,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  mddo-toolbox-cli (>= 1.2.0)!
+  mddo-toolbox-cli (>= 1.2.1)!
+  netomox (>= 0.9.1)!
 
 BUNDLED WITH
    2.4.12

demo/copy_to_emulated_env/.gitignore

@@ -1,4 +1,2 @@
 # files generated in demo scenario
 *.bak
-*_ext.json
-

demo/copy_to_emulated_env/demo_remove.sh

@@ -10,12 +10,13 @@ fi
 source ./demo_vars
 
 # delete files and containers related to containerlab
-rm -f "$ANSIBLE_RUNNER_DIR/project/playbooks/configs/"*.conf
-sudo containerlab destroy --topo "$ANSIBLE_RUNNER_DIR/clab/clab-topo.yaml" --cleanup
-sudo rm -f "$ANSIBLE_RUNNER_DIR/clab/"*.conf
-sudo rm -f "$ANSIBLE_RUNNER_DIR/clab/clab-topo.yaml"
+rm -f "${USECASE_CONFIGS_DIR}"/*.conf
+sudo containerlab destroy --topo "${ANSIBLE_RUNNER_DIR}/clab/clab-topo.yaml" --cleanup
+sudo rm -f "${ANSIBLE_RUNNER_DIR}/clab"/*.conf
+sudo rm -f "${ANSIBLE_RUNNER_DIR}/clab/clab-topo.yaml"
 
 # delete ovs bridges
 curl -s "http://${API_PROXY}/topologies/${NETWORK_NAME}/emulated_asis/topology/layer3/nodes?node_type=segment" \
   | jq '.nodes[].alias.l1_principal' \
   | xargs -I@ sudo ovs-vsctl del-br @
+

demo/copy_to_emulated_env/demo_step1-2.sh

@@ -10,13 +10,6 @@ else
   echo "# Network:$NETWORK_NAME is specified as BGP network, expand external-AS network and splice it into topology data"
 fi
 
-# for PNI-addlink (janog53) demo
-# generate "external-as topology script"
-if  [[ "$USECASE_NAME" == "pni_addlink" ]]; then
-  python3 ./generate.py "${NETWORK_NAME}" "${USECASE_NAME}" "${SOURCE_AS}" "${DEST_AS}" "169.254.0.0/16" "${PREFERRED_NODE}" "${REDUNDANT_NODE}"
-  # python3 ./generate.py "${NETWORK_NAME}" "${SOURCE_AS}" "${DEST_AS}" "100.0.0.0/8"
-fi
-
 # bgp-policy data handling
 # parse configuration files with TTP
 curl -s -X POST -H "Content-Type: application/json" \
@@ -29,11 +22,15 @@ curl -s -X POST -H "Content-Type: application/json" \
   "http://${API_PROXY}/bgp_policy/${NETWORK_NAME}/original_asis/topology"
 
 # external-AS data handling
+# generate "external-as topology script" for PNI-addlink (janog53) demo
+if  [[ "$USECASE_NAME" == "pni_addlink" ]]; then
+  echo "# NOTE: interim operation to generate external-as script for pni_addlink usecase"
+  python3 "${USECASE_DIR}/generate.py" -p "${USECASE_DIR}/params.yaml"
+fi
+
 # generate external-AS topology
-external_as_json="${NETWORK_NAME}_ext.json"
-curl -s -X POST -H "Content-type: application/json" \
-  -d "{ \"usecase\": \"${USECASE_NAME}\", \"options\": {} }" \
-  "http://${API_PROXY}/topologies/${NETWORK_NAME}/original_asis/external_as_topology" > "$external_as_json"
+external_as_json="${USECASE_CONFIGS_DIR}/external_as_topology.json"
+ruby "${USECASE_DIR}/external_as_topology/main.rb" -p "${USECASE_DIR}/params.yaml" > "$external_as_json"
 # splice external-AS topology to original_asis (overwrite)
 curl -s -X POST -H "Content-Type: application/json" \
   -d @<(jq '{ "overwrite": true, "ext_topology_data": . }' "$external_as_json") \

demo/copy_to_emulated_env/demo_step2-1.sh

@@ -10,18 +10,33 @@
 # shellcheck disable=SC1091
 source ./demo_vars
 
+print_usage() {
+    echo "Usage: $(basename "$0") [options]"
+    echo "Options:"
+    echo "  -d     Debug/data check, without executing ansible-runner (clab)"
+    echo "  -h     Display this help message"
+}
+
 # option check
-while getopts d option; do
+# defaults
+WITH_CLAB=true
+while getopts dh option; do
   case $option in
   d)
     # data check, debug
     # -> without container lab; does not build emulated-env
     WITH_CLAB=false
     echo "# Check: with_clab = $WITH_CLAB"
     ;;
+  h)
+    print_usage
+    exit 0
+    ;;
   *)
-    echo "Unknown option detected, $option"
+    echo "Unknown option detected, -$OPTARG" >&2
+    print_usage
     exit 1
+    ;;
   esac
 done
 
@@ -31,11 +46,18 @@ curl -s -X POST -H 'Content-Type: application/json' \
   "http://${API_PROXY}/conduct/${NETWORK_NAME}/ns_convert/original_asis/emulated_asis"
 
 # generate emulated asis configs from emulated asis topology
-if "${WITH_CLAB:-true}"; then
-  ansible-runner run . -p /data/project/playbooks/step2-1.yaml --container-option="--net=${API_BRIDGE}" \
-    --container-volume-mount="$PWD:/data" --container-image="${ANSIBLE_RUNNER_IMAGE}" \
-    --process-isolation --process-isolation-executable docker \
-    --cmdline "-e ansible_runner_dir=${ANSIBLE_RUNNER_DIR} -e login_user=${LOCALSERVER_USER} -e network_name=${NETWORK_NAME} -e crpd_image=${CRPD_IMAGE} -k -K "
+if "${WITH_CLAB}"; then
+  ansible-runner run . -p /data/project/playbooks/step2-1.yaml \
+    --container-option="--net=${API_BRIDGE}" \
+    --container-image="${ANSIBLE_RUNNER_IMAGE}" \
+    --container-volume-mount="$PWD:/data" \
+    --process-isolation \
+    --process-isolation-executable docker \
+    --cmdline "-e ansible_runner_dir=${ANSIBLE_RUNNER_DIR} \
+               -e login_user=${LOCALSERVER_USER} \
+               -e network_name=${NETWORK_NAME} \
+               -e crpd_image=${CRPD_IMAGE} \
+               -k -K"
 fi
 
 # update netoviz index

demo/copy_to_emulated_env/demo_step2-2.sh

@@ -10,43 +10,64 @@
 # shellcheck disable=SC1091
 source ./demo_vars
 
+print_usage() {
+    echo "Usage: $(basename "$0") [options]"
+    echo "Options:"
+    echo "  -d     Debug/data check, without executing ansible-runner (clab)"
+    echo "  -r     Restart containerlab"
+    echo "  -h     Display this help message"
+}
+
 # option check
-while getopts d option; do
+# defaults
+WITH_CLAB=true
+CLAB_RESTART=false
+while getopts drh option; do
   case $option in
   d)
     # data check, debug
     # -> without container lab; does not build emulated-env
     WITH_CLAB=false
     echo "# Check: with_clab = $WITH_CLAB"
     ;;
+  r)
+    # restart clab/iperf
+    CLAB_RESTART=true
+    ;;
+  h)
+    print_usage
+    exit 0
+    ;;
   *)
-    echo "Unknown option detected, $option"
+    echo "Unknown option detected, -$OPTARG" >&2
+    print_usage
     exit 1
+    ;;
   esac
 done
 
-# convert node/interface name (original to emulated)
-EMULATE_PREFERRED_NODE=$(
-  curl -s -X POST -H 'Content-Type: application/json' \
-    -d "{ \"host_name\": \"${PREFERRED_NODE}\" }" \
-    "http://${API_PROXY}/topologies/${NETWORK_NAME}/ns_convert_table/query" | jq -r .target_host.l3_model
-)
-EMULATE_PREFERRED_INTERFACE=$(
-  curl -s -X POST -H 'Content-Type: application/json' \
-    -d "{ \"host_name\": \"${PREFERRED_NODE}\", \"if_name\": \"${PREFERRED_INTERFACE}\" }" \
-    "http://${API_PROXY}/topologies/${NETWORK_NAME}/ns_convert_table/query" | jq -r .target_if.l3_model
-)
-
-# set preferred peer
-curl -s -X POST -H "Content-Type: application/json" \
-  -d "{ \"ext_asn\": ${EXTERNAL_ASN}, \"node\": \"${EMULATE_PREFERRED_NODE}\", \"interface\": \"${EMULATE_PREFERRED_INTERFACE}\" }" \
-  "http://${API_PROXY}/conduct/${NETWORK_NAME}/emulated_asis/topology/bgp_proc/preferred_peer" \
-  > /dev/null # ignore echo-back (topology json)
+if [[ ! "$NETWORK_NAME" =~ $BGP_NETWORK_PATTERN ]]; then
+  echo "Network:$NETWORK_NAME is not BGP network (Nothing to do in step2-2)"
+  exit 0
+else
+  echo "# Network:$NETWORK_NAME is specified as BGP network, generate traffic between PNI and POI"
+fi
 
 # configure iperf client/server
-if "${WITH_CLAB:-true}"; then
-  ansible-runner run . -p /data/project/playbooks/step2-2.yaml --container-option="--net=${API_BRIDGE}" \
-    --container-volume-mount="$PWD:/data" --container-image="${ANSIBLE_RUNNER_IMAGE}" \
-    --process-isolation --process-isolation-executable docker \
-    --cmdline "-e ansible_runner_dir=${ANSIBLE_RUNNER_DIR} -e playground_dir=${PLAYGROUND_DIR} -e login_user=${LOCALSERVER_USER} -e network_name=${NETWORK_NAME} -k -K "
+if "${WITH_CLAB}"; then
+  echo "# Check: clab_restart = $CLAB_RESTART"
+
+  ansible-runner run . -p "/data/project/playbooks/step2-2.yaml" \
+    --container-option="--net=${API_BRIDGE}" \
+    --container-image="${ANSIBLE_RUNNER_IMAGE}" \
+    --container-volume-mount="$PWD:/data" \
+    --process-isolation \
+    --process-isolation-executable docker \
+    --cmdline "-e ansible_runner_dir=${ANSIBLE_RUNNER_DIR} \
+               -e login_user=${LOCALSERVER_USER} \
+               -e network_name=${NETWORK_NAME} \
+               -e usecase_name=${USECASE_NAME} \
+               -e usecase_common_name=${USECASE_COMMON_NAME} \
+               -e clab_restart=${CLAB_RESTART} \
+               -k -K"
 fi

demo/copy_to_emulated_env/demo_vars

@@ -1,27 +1,23 @@
+# runtime and environment
 ANSIBLE_RUNNER_IMAGE="ghcr.io/ool-mddo/mddo-ansible-runner:v3.1.0"
 CRPD_IMAGE="crpd:23.4R1.9"
 API_PROXY="localhost:15000"
 API_BRIDGE="playground_default"
 LOCALSERVER_USER=mddo
-PLAYGROUND_DIR="/home/${LOCALSERVER_USER}/playground"
-ANSIBLE_RUNNER_DIR="${PLAYGROUND_DIR}/demo/copy_to_emulated_env"
 
-# all steps: target network name
+# all steps: target network/usecase name
 NETWORK_NAME="mddo-bgp"
 USECASE_NAME="pni_addlink" # "pni_addlink" or "pni_te" for mddo-bgp network
+
+# constants
 NETWORK_INDEX="network_index/${NETWORK_NAME}.json"
+PLAYGROUND_DIR="/home/${LOCALSERVER_USER}/playground"
+ANSIBLE_RUNNER_DIR="${PLAYGROUND_DIR}/demo/copy_to_emulated_env"
+ANSIBLE_PLAYBOOK_DIR="${ANSIBLE_RUNNER_DIR}/project/playbooks"
+USECASE_CONFIGS_DIR="${ANSIBLE_PLAYBOOK_DIR}/configs"
+USECASE_COMMON_NAME=$(echo "$USECASE_NAME" | cut -d_ -f1)
+USECASE_COMMON_DIR="${ANSIBLE_PLAYBOOK_DIR}/${USECASE_COMMON_NAME}"
+USECASE_DIR="${ANSIBLE_PLAYBOOK_DIR}/${USECASE_NAME}"
 
 # step1-2: define network-name regexp (to merge bgp-policy data/external-as topology)
 BGP_NETWORK_PATTERN="^(biglobe|mddo-bgp).*$"
-
-# step1-2, generate external-AS topology script
-SOURCE_AS=65550
-DEST_AS=65520
-
-# step1-2 and 2-2, preffered peer parameter (use original_asis node/interface name)
-PREFERRED_NODE="NONE"
-#PREFERRED_NODE="edge-tk01"
-#REDUNDANT_NODE="edge-tk03"
-REDUNDANT_NODE="NONE"
-PREFERRED_INTERFACE="ge-0/0/3.0"
-EXTERNAL_ASN=65550