Skip to content

Commit

Permalink
Add fake multinode system tests.
Browse files Browse the repository at this point in the history
This patch adds a simple system test using ovn-fake-multinode
setup.  The tests can be run as - 'make check-multinode'

Before running these tests, user should deploy fake_multinode setup
by running 'ovn_cluster.sh start'.

This test suite is also triggered for the newly added fake multinode CI
job.

The fake multinode system tests suite can be enhanced further for new
features and to cover multi node scenarios.

Reviewed-by: Simon Horman <simon.horman@corigine.com>
Acked-by: Dumitru Ceara <dceara@redhat.com>
Signed-off-by: Numan Siddique <numans@ovn.org>
  • Loading branch information
numansiddique committed May 2, 2023
1 parent d245dcc commit 0ab0580
Show file tree
Hide file tree
Showing 5 changed files with 354 additions and 4 deletions.
64 changes: 61 additions & 3 deletions .github/workflows/ovn-fake-multinode-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,13 @@ jobs:
CENTRAL_IMAGE: ${{ matrix.cfg.central_image }}
# Disable SSL for now. Revisit this if required.
ENABLE_SSL: no
CC: gcc
OPTS: "--disable-ssl"
dependencies: |
automake libtool gcc bc libjemalloc2 libjemalloc-dev \
libssl-dev llvm-dev libelf-dev libnuma-dev libpcap-dev \
selinux-policy-dev ncat python3-scapy isc-dhcp-server \
podman openvswitch-switch libunbound-dev libunwind-dev
# https://github.com/actions/runner-images/issues/6282
XDG_RUNTIME_DIR: ''

Expand All @@ -105,6 +112,8 @@ jobs:
- { central_image: "ovn/ovn-multi-node:22.03" }

steps:
- name: install required dependencies
run: sudo apt install -y ${{ env.dependencies }}

- name: Free up disk space
run: sudo eatmydata apt-get remove --auto-remove -y aspnetcore-* dotnet-* libmono-* mono-* msbuild php-* php7* ghc-* zulu-*
Expand All @@ -129,10 +138,8 @@ jobs:
path: 'ovn-fake-multinode'
ref: 'v0.1'

- name: Install dependencies
- name: Start openvswitch
run: |
sudo apt update
sudo apt-get install -y podman openvswitch-switch
sudo systemctl start openvswitch-switch
sudo ovs-vsctl show
Expand All @@ -149,6 +156,57 @@ jobs:
sudo ./.ci/test_basic.sh
working-directory: ovn-fake-multinode

- name: update PATH
run: |
echo "$HOME/bin" >> $GITHUB_PATH
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.x'

- name: Check out ovn
uses: actions/checkout@v3
with:
path: 'ovn'
submodules: recursive

- name: Build OVN and trigger fake-multinode system tests
run: |
set -x
pwd
cd ovn
./.ci/linux-build.sh
sudo make check-multinode || :
sudo podman exec -it ovn-central ovn-nbctl show || :
sudo podman exec -it ovn-central ovn-sbctl show || :
sudo podman exec -it ovn-chassis-1 ovs-vsctl show || :
sudo podman exec -it ovn-chassis-1 ip netns || :
sudo podman exec -it ovn-chassis-1 cat /var/log/ovn/ovn-controller.log || :
sudo cat tests/multinode-testsuite.dir/1/multinode-testsuite.log || :
- name: copy logs on failure
if: failure() || cancelled()
run: |
# upload-artifact@v3 throws exceptions if it tries to upload socket
# files and we could have some socket files in testsuite.dir.
# Also, upload-artifact@v3 doesn't work well enough with wildcards.
# So, we're just archiving everything here to avoid any issues.
mkdir logs
cp ovn/config.log ./logs/
# multinode tests are run as root, need to adjust permissions.
sudo chmod -R +r ovn/tests/multinode-testsuite.dir.* || true
cp -r ovn/tests/multinode-testsuite.dir.* ./logs/ || true
tar -czvf logs.tgz logs/
- name: upload logs on failure
if: failure() || cancelled()
uses: actions/upload-artifact@v3
with:
name: logs-linux-${{ join(matrix.cfg.*, '-') }}
path: logs.tgz

- name: Stop cluster
run: |
sudo -E ./ovn_cluster.sh stop
Expand Down
27 changes: 26 additions & 1 deletion tests/automake.mk
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ EXTRA_DIST += \
$(SYSTEM_KMOD_TESTSUITE_AT) \
$(SYSTEM_USERSPACE_TESTSUITE_AT) \
$(PERF_TESTSUITE_AT) \
$(MULTINODE_TESTSUITE_AT) \
$(TESTSUITE) \
$(SYSTEM_KMOD_TESTSUITE) \
$(SYSTEM_USERSPACE_TESTSUITE) \
$(PERF_TESTSUITE) \
$(MULTINODE_TESTSUITE) \
tests/atlocal.in \
$(srcdir)/package.m4 \
$(srcdir)/tests/testsuite \
Expand Down Expand Up @@ -61,6 +63,11 @@ PERF_TESTSUITE_AT = \
tests/perf-testsuite.at \
tests/perf-northd.at

MULTINODE_TESTSUITE_AT = \
tests/multinode-testsuite.at \
tests/multinode-macros.at \
tests/multinode.at

check_SCRIPTS += tests/atlocal

TESTSUITE = $(srcdir)/tests/testsuite
Expand All @@ -72,7 +79,9 @@ PERF_TESTSUITE = $(srcdir)/tests/perf-testsuite
PERF_TESTSUITE_DIR = $(abs_top_builddir)/tests/perf-testsuite.dir
PERF_TESTSUITE_RESULTS = $(PERF_TESTSUITE_DIR)/results
DISTCLEANFILES += tests/atconfig tests/atlocal

MULTINODE_TESTSUITE = $(srcdir)/tests/multinode-testsuite
MULTINODE_TESTSUITE_DIR = $(abs_top_builddir)/tests/multinode-testsuite.dir
MULTINODE_TESTSUITE_RESULTS = $(MULTINODE_TESTSUITE_DIR)/results
AUTOTEST_PATH = $(ovs_builddir)/utilities:$(ovs_builddir)/vswitchd:$(ovs_builddir)/ovsdb:$(ovs_builddir)/vtep:tests:$(PTHREAD_WIN32_DIR_DLL):$(SSL_DIR):controller-vtep:northd:utilities:controller:ic

export ovs_srcdir
Expand Down Expand Up @@ -197,6 +206,18 @@ check-perf: all
@echo
@echo "Results can be found in $(PERF_TESTSUITE_RESULTS)"

check-multinode: all
@mkdir -p $(MULTINODE_TESTSUITE_DIR)
@echo > $(MULTINODE_TESTSUITE_RESULTS)
set $(SHELL) '$(MULTINODE_TESTSUITE)' -C tests AUTOTEST_PATH='$(AUTOTEST_PATH)'; \
"$$@" $(TESTSUITEFLAGS) -j1 || (test X'$(RECHECK)' = Xyes && "$$@" --recheck)
@echo
@echo '## -------------------- ##'
@echo '## Multinode test Results ##'
@echo '## -------------------- ##'
@cat $(MULTINODE_TESTSUITE_RESULTS)
@echo
@echo "Results can be found in $(MULTINODE_TESTSUITE_RESULTS)"

AUTOTEST = $(AUTOM4TE) --language=autotest

Expand All @@ -223,6 +244,10 @@ $(PERF_TESTSUITE): package.m4 $(PERF_TESTSUITE_AT) $(COMMON_MACROS_AT)
$(AM_V_GEN)$(AUTOTEST) -I '$(srcdir)' -o $@.tmp $@.at
$(AM_V_at)mv $@.tmp $@

$(MULTINODE_TESTSUITE): package.m4 $(MULTINODE_TESTSUITE_AT) $(COMMON_MACROS_AT)
$(AM_V_GEN)$(AUTOTEST) -I '$(srcdir)' -o $@.tmp $@.at
$(AM_V_at)mv $@.tmp $@

# The `:;' works around a Bash 3.2 bug when the output is not writeable.
$(srcdir)/package.m4: $(top_srcdir)/configure.ac
$(AM_V_GEN):;{ \
Expand Down
166 changes: 166 additions & 0 deletions tests/multinode-macros.at
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
#
#

# M_NS_EXEC([fake_node], [namespace], [command])
#
# Execute 'command' in 'namespace'
m4_define([M_NS_EXEC],
[podman exec $1 ip netns exec $2 $3])

# M_NS_CHECK_EXEC([fake_node], [namespace], [command], other_params...)
#
# Wrapper for AT_CHECK that executes 'command' inside 'fake_node''s namespace'.
# 'other_params' as passed as they are to AT_CHECK.
m4_define([M_NS_CHECK_EXEC],
[ AT_CHECK([M_NS_EXEC([$1], [$2], [$3])], m4_shift(m4_shift(m4_shift($@)))) ]
)

OVS_START_SHELL_HELPERS

m_as() {
c=$1
shift
podman exec $c "$@"
}

m_central_as () {
podman exec ovn-central "$@"
}

check_fake_multinode_setup() {
check m_as ovn-central ovn-nbctl --wait=sb sync
AT_CHECK([m_as ovn-chassis-1 ovn-appctl -t ovn-controller version], [0], [ignore])
AT_CHECK([m_as ovn-chassis-2 ovn-appctl -t ovn-controller version], [0], [ignore])
AT_CHECK([m_as ovn-gw-1 ovn-appctl -t ovn-controller version], [0], [ignore])
AT_CHECK([m_as ovn-gw-1 ovn-appctl -t ovn-controller version], [0], [ignore])
}

cleanup_multinode_resources() {
m_as ovn-central rm -f /etc/ovn/ovnnb_db.db
m_as ovn-central /usr/share/ovn/scripts/ovn-ctl restart_northd
check m_as ovn-central ovn-nbctl --wait=sb sync
for c in ovn-chassis-1 ovn-chassis-2 ovn-gw-1
do
m_as $c ovs-vsctl del-br br-int
m_as $c ip --all netns delete
done
}

multinode_nbctl () {
m_as ovn-central ovn-nbctl "$@"
}

# m_count_rows TABLE [CONDITION...]
#
# Prints the number of rows in TABLE (that satisfy CONDITION).
# Uses the southbound db by default; set DB=nb for the northbound database.
m_count_rows() {
local db=$(parse_db $1) table=$(parse_table $1); shift
m_central_as ovn-${db}ctl --format=table --no-headings find $table "$@" | wc -l
}

# m_check_row_count [DATABASE:]TABLE COUNT [CONDITION...]
#
# Checks that TABLE contains COUNT rows (that satisfy CONDITION).
# The default DATABASE is "sb".
m_check_row_count() {
local db=$(parse_db $1) table=$(parse_table $1); shift
local count=$1; shift
local found=$(m_count_rows $c $db:$table "$@")
echo
echo "Checking for $count rows in $db $table${1+ with $*}... found $found"
if test "$count" != "$found"; then
m_central_as ovn-${db}ctl list $table
AT_FAIL_IF([:])
fi
}

# m_wait_row_count [DATABASE:]TABLE COUNT [CONDITION...]
#
# Waits until TABLE contains COUNT rows (that satisfy CONDITION).
# The default DATABASE is "sb".
m_wait_row_count() {
local db=$(parse_db $1) table=$(parse_table $1); shift
local count=$1; shift
local a=$1 b=$2 c=$3 d=$4 e=$5
echo "Waiting until $count rows in $db $table${1+ with $*}..."
OVS_WAIT_UNTIL([test $count = $(m_count_rows $db:$table $a $b $c $d $e)],[
echo "$db table $table has the following rows. $(m_count_rows $db:$table $a $b $c $d $e) rows match instead of expected $count:"
m_central_as ovn-${db}ctl list $table])
}

# m_wait_column EXPECTED [DATABASE:]TABLE [COLUMN [CONDITION...]]
#
# Wait until all of the values of COLUMN in the rows of TABLE (that
# satisfy CONDITION) equal EXPECTED (ignoring order).
#
# The default DATABASE is "sb".
#
# COLUMN defaults to _uuid if unspecified.
m_wait_column() {
local expected=$(for d in $1; do echo $d; done | sort)
local db=$(parse_db $2) table=$(parse_table $2) column=${3-_uuid}; shift; shift; shift
local a=$1 b=$2 c=$3 d=$4 e=$5

echo
echo "Waiting until $column in $db $table${1+ with $*} is $expected..."
OVS_WAIT_UNTIL([
found=$(m_central_as ovn-${db}ctl --bare --columns $column find $table $a $b $c $d $e)
found=$(for d in $found; do echo $d; done | sort)
test "$expected" = "$found"
], [
echo "$column in $db table $table has value $found, from the following rows:"
m_central_as ovn-${db}ctl list $table])
}

# m_fetch_column [DATABASE:]TABLE COLUMN [CONDITION...]
#
# Fetches and prints all the values of COLUMN in the rows of TABLE
# (that satisfy CONDITION), sorting the results lexicographically.
# The default DATABASE is "sb".
m_fetch_column() {
local db=$(parse_db $1) table=$(parse_table $1) column=${2-_uuid}; shift; shift
# Using "echo" removes spaces and newlines.
echo $(m_central_as ovn-${db}ctl --bare --columns $column find $table "$@" | sort)
}

# m_check_column EXPECTED [DATABASE:]TABLE COLUMN [CONDITION...]
#
# Fetches all of the values of COLUMN in the rows of TABLE (that
# satisfy CONDITION), and compares them against EXPECTED (ignoring
# order).
#
# The default DATABASE is "sb".
m_check_column() {
local expected=$1 db=$(parse_db $2) table=$(parse_table $2) column=${3-_uuid}; shift; shift; shift
local found=$(m_central_as ovn-${db}ctl --bare --columns $column find $table "$@")

# Sort the expected and found values.
local found=$(for d in $found; do echo $d; done | sort)
local expected=$(for d in $expected; do echo $d; done | sort)

echo
echo "Checking values in $db $table${1+ with $*} against $expected... found $found"
if test "$found" != "$expected"; then
m_central_as ovn-${db}ctl list $table
AT_FAIL_IF([:])
fi
}

# m_wait_for_ports_up [PORT...]
#
# With arguments, waits for specified Logical_Switch_Ports to come up.
# Without arguments, waits for all "plain" and router
# Logical_Switch_Ports to come up.
m_wait_for_ports_up() {
if test $# = 0; then
m_wait_row_count nb:Logical_Switch_Port 0 up!=true type='""'
m_wait_row_count nb:Logical_Switch_Port 0 up!=true type=router
else
for port; do
m_wait_row_count nb:Logical_Switch_Port 1 up=true name=$port
done
fi
}

OVS_END_SHELL_HELPERS
27 changes: 27 additions & 0 deletions tests/multinode-testsuite.at
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
AT_INIT

AT_COPYRIGHT([Copyright (c) 2022 Red Hat,

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.])

m4_ifdef([AT_COLOR_TESTS], [AT_COLOR_TESTS])
AT_ARG_OPTION([rebuild], [Do not use cached versions of databases])

m4_include([tests/ovs-macros.at])
m4_include([tests/ovsdb-macros.at])
m4_include([tests/ofproto-macros.at])
m4_include([tests/ovn-macros.at])
m4_include([tests/system-common-macros.at])
m4_include([tests/multinode-macros.at])

m4_include([tests/multinode.at])
Loading

0 comments on commit 0ab0580

Please sign in to comment.