Bring silk src into silk-release

packages/silk-cni/spec

@@ -28,16 +28,16 @@ files:
   - code.cloudfoundry.org/lib/rules/*.go # gosub-main-module
   - code.cloudfoundry.org/lib/serial/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/code.cloudfoundry.org/policy_client/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cmd/silk-cni/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cni/adapter/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cni/config/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cni/lib/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cni/netinfo/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/daemon/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/adapter/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/datastore/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/hwaddr/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/serial/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cmd/silk-cni/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cni/adapter/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cni/config/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cni/lib/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cni/netinfo/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/daemon/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/adapter/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/datastore/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/hwaddr/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/serial/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/alexflint/go-filemutex/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/containernetworking/cni/pkg/invoke/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/containernetworking/cni/pkg/skel/*.go # gosub-main-module

packages/silk-controller/spec

@@ -20,14 +20,14 @@ files:
   - code.cloudfoundry.org/vendor/code.cloudfoundry.org/lager/v3/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/code.cloudfoundry.org/lager/v3/internal/truncate/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/code.cloudfoundry.org/lager/v3/lagerflags/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cmd/silk-controller/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/config/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/database/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/handlers/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/leaser/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/server_metrics/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/hwaddr/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cmd/silk-controller/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/config/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/database/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/handlers/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/leaser/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/server_metrics/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/hwaddr/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/bmizerany/pat/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/cloudfoundry/dropsonde/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/cloudfoundry/dropsonde/emitter/*.go # gosub-main-module

packages/silk-daemon/spec

@@ -28,17 +28,17 @@ files:
   - code.cloudfoundry.org/silk-daemon-bootstrap/*.go # gosub-main-module
   - code.cloudfoundry.org/silk-daemon-bootstrap/config/*.go # gosub-main-module
   - code.cloudfoundry.org/silk-daemon-shutdown/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/client/config/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cmd/silk-daemon/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/cmd/silk-teardown/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/controller/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/daemon/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/daemon/planner/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/daemon/poller/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/daemon/vtep/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/adapter/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/datastore/*.go # gosub-main-module
-  - code.cloudfoundry.org/vendor/code.cloudfoundry.org/silk/lib/serial/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/client/config/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cmd/silk-daemon/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/cmd/silk-teardown/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/controller/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/daemon/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/daemon/planner/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/daemon/poller/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/daemon/vtep/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/adapter/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/datastore/*.go # gosub-main-module
+  - code.cloudfoundry.org/silk/lib/serial/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/cloudfoundry/dropsonde/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/cloudfoundry/dropsonde/emitter/*.go # gosub-main-module
   - code.cloudfoundry.org/vendor/github.com/cloudfoundry/dropsonde/envelope_sender/*.go # gosub-main-module

scripts/docker-shell

@@ -20,7 +20,8 @@ docker run \
    --rm \
    -it \
    --privileged \
-   -v $PWD:/silk \
-   -w /silk \
-   $docker_image \
-   /bin/bash "$@"
+   -v "${PWD}:/silk-release" \
+   -e "DB=${db}" \
+   -w /silk-release \
+   "${docker_image}" \
+   /bin/bash "${@}"

scripts/start-db-helper

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+function bootDB {
+  db="$1"
+
+  if [ "${db}" = "postgres" ]; then
+    launchDB="(/postgres-entrypoint.sh postgres &> /var/log/postgres-boot.log) &"
+    testConnection="psql -h localhost -U postgres -c '\conninfo'"
+  elif [ "${db}" = "mysql" ]  || [ "${db}" = "mysql-5.6" ] || [ "${db}" = "mysql8" ]; then
+    launchDB="(MYSQL_ROOT_PASSWORD=password /mysql-entrypoint.sh mysqld &> /var/log/mysql-boot.log) &"
+    testConnection="mysql -h localhost -u root -D mysql -e '\s;' --password='password'"
+  else
+    echo "skipping database"
+    return 0
+  fi
+
+  echo -n "booting ${db}"
+  eval "$launchDB"
+  for _ in $(seq 1 60); do
+    if eval "${testConnection}" &> /dev/null; then
+      echo "connection established to ${db}"
+      return 0
+    fi
+    echo -n "."
+    sleep 1
+  done
+  eval "${testConnection}" || true
+  echo "unable to connect to ${db}"
+  exit 1
+}

scripts/test.sh

@@ -1,74 +1,104 @@
 #!/bin/bash
 
-set -eu
-set -o pipefail
+specificied_package="${1}"
+
+set -e -u
 
 go version # so we see the version tested in CI
 
-# In the cf-networking-and-silk-pr.yml pipeline, we need to run db-unit tests for cf-networking, but
-# concourse doesn't have a way of conditionally adding jobs, so it will end up running db-unit tests
-# against silk, which doesn't do anything other than run unit tests again, so we skip it here.
-if [[ -n "${DB:-""}" ]]; then
-  echo "No DB specific silk tests have been defined. Skipping this step."
-  exit 0
-fi
+SCRIPT_PATH="$(cd "$(dirname "${0}")" && pwd)"
+. "${SCRIPT_PATH}/start-db-helper"
+
+cd "${SCRIPT_PATH}/.."
 
-cd $(dirname $0)/..
+DB="${DB:-"notset"}"
+
+## Setting to other than 1 node will break cni-wrapper-plugin/integration
+serial_nodes=1
 
 declare -a serial_packages=(
     "src/code.cloudfoundry.org/cni-wrapper-plugin/integration"
-    "src/code.cloudfoundry.org/vxlan-policy-agent/integration/linux"
-    "src/code.cloudfoundry.org/silk-daemon-shutdown/integration"
     "src/code.cloudfoundry.org/silk-daemon-bootstrap/integration"
+    "src/code.cloudfoundry.org/silk-daemon-shutdown/integration"
+    "src/code.cloudfoundry.org/silk/cni/integration"
+    "src/code.cloudfoundry.org/vxlan-policy-agent/integration/linux"
     )
 
 declare -a windows_packages=(
     "src/code.cloudfoundry.org/vxlan-policy-agent/integration/windows"
     )
 
-# get all git submodule paths | print only the path without the extra info | cut the "package root" for go | deduplicate
-declare -a git_modules=($(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | cut -d'/' -f1,2 | sort -u))
+declare -a ignored_packages
 
-declare -a packages=($(find src -type f -name "*_test.go" | xargs -L 1 -I{} dirname {} | sort -u))
+# gather ignored packages from exclude_packages
+for pkg in $(echo "${exclude_packages:-""}" | jq -r .[]); do
+  ignored_packages+=("${pkg}")
+done
 
+# gather more ignored packages because they are windows code
+for pkg in "${windows_packages[@]}"; do
+  ignored_packages+=("${pkg}")
+done
+
+containsElement() {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+test_package() {
+  local package=$1
+  if [ ! -d "${package}" ]; then
+    return 0
+  fi
+  shift
+  pushd "${package}" &>/dev/null
+  pwd
+  go run github.com/onsi/ginkgo/v2/ginkgo --race -randomize-all -randomize-suites -fail-fast \
+      -ldflags="extldflags=-WL,--allow-multiple-definition" \
+       "${@}";
+  rc=$?
+  popd &>/dev/null
+  return "${rc}"
+}
+
+bootDB "${DB}"
+
+declare -a packages
+if [[ -n "${include_only:-""}" ]]; then
+  mapfile -t packages < <(echo "${include_only}" | jq -r .[])
+else
+  mapfile -t packages < <(find src -type f -name '*_test.go' -print0 | xargs -0 -L1 -I{} dirname {} | sort -u)
+fi
 
 # filter out serial_packages from packages
 for i in "${serial_packages[@]}"; do
-  packages=(${packages[@]//*$i*})
+  packages=("${packages[@]//*$i*}")
 done
 
-# filter out windows_packages from packages
-for i in "${windows_packages[@]}"; do
-  packages=(${packages[@]//*$i*})
+# filter out explicitly ignored packages
+for i in "${ignored_packages[@]}"; do
+  packages=("${packages[@]//*$i*}")
+  serial_packages=("${serial_packages[@]//*$i*}")
 done
 
-if [ "${1:-""}" = "" ]; then
+if [[ -z "${specificied_package}" ]]; then
+  echo "testing packages: " "${packages[@]}"
   for dir in "${packages[@]}"; do
-    pushd "$dir"
-      go run github.com/onsi/ginkgo/v2/ginkgo -p --race --randomize-all --randomize-suites \
-        -ldflags="-extldflags=-Wl,--allow-multiple-definition" \
-        ${@:2}
-    popd
+    test_package "${dir}" -p
   done
+  echo "testing serial packages: " "${serial_packages[@]}"
   for dir in "${serial_packages[@]}"; do
-    pushd "$dir"
-      go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --fail-fast \
-        -ldflags="-extldflags=-Wl,--allow-multiple-definition" \
-        ${@:2}
-    popd
+    test_package "${dir}" --nodes "${serial_nodes}"
   done
 else
-  dir="${@: -1}"
-  dir="${dir#./}"
-  for package in "${serial_packages[@]}"; do
-    if [[ "${dir##$package}" != "${dir}" ]]; then
-      go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --fail-fast \
-        -ldflags="-extldflags=-Wl,--allow-multiple-definition" \
-        "${@}"
-      exit $?
-    fi
-  done
-  go run github.com/onsi/ginkgo/v2/ginkgo -p --race --randomize-all --randomize-suites --fail-fast --skip-package windows \
-    -ldflags="-extldflags=-Wl,--allow-multiple-definition" \
-    "${@}"
+  specificied_package="${specificied_package#./}"
+  if containsElement "${specificied_package}" "${serial_packages[@]}"; then
+    echo "testing serial package ${specificied_package}"
+    test_package "${specificied_package}" --nodes "${serial_nodes}"
+  else
+    echo "testing package ${specificied_package}"
+    test_package "${specificied_package}" -p
+  fi
 fi

src/code.cloudfoundry.org/go.mod

@@ -17,19 +17,25 @@ require (
 	code.cloudfoundry.org/lager/v3 v3.0.2
 	code.cloudfoundry.org/policy_client v0.0.0-20230726190751-c4580e1b1f80
 	code.cloudfoundry.org/runtimeschema v0.0.0-20230323223330-5366865eed76
-	code.cloudfoundry.org/silk v0.0.0-20230728161150-8e48b8d1e681
 	github.com/cloudfoundry/dropsonde v1.1.0
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.3.0
 	github.com/coreos/go-iptables v0.6.0
+	github.com/go-sql-driver/mysql v1.7.1
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hpcloud/tail v1.0.0
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/lib/pq v1.10.9
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.10
 	github.com/pivotal-cf-experimental/gomegamatchers v0.0.0-20180326192815-e36bfcc98c3a
+	github.com/rubenv/sql-migrate v1.5.2
 	github.com/tedsuo/ifrit v0.0.0-20230516164442-7862c310ad26
+	github.com/tedsuo/rata v1.0.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
+	github.com/ziutek/utils v0.0.0-20190626152656-eb2a3b364d6c
 	gopkg.in/validator.v2 v2.0.1
 )
 
@@ -46,27 +52,20 @@ require (
 	github.com/cloudfoundry/sonde-go v0.0.0-20230710164515-a0a43d1dbbf8 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/go-test/deep v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/jackc/pgx v3.6.2+incompatible // indirect
-	github.com/jmoiron/sqlx v1.3.5 // indirect
-	github.com/lib/pq v1.10.9 // indirect
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
 	github.com/openzipkin/zipkin-go v0.4.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rubenv/sql-migrate v1.5.2 // indirect
 	github.com/safchain/ethtool v0.3.0 // indirect
 	github.com/square/certstrap v1.3.0 // indirect
-	github.com/tedsuo/rata v1.0.0 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
-	github.com/ziutek/utils v0.0.0-20190626152656-eb2a3b364d6c // indirect
 	go.step.sm/crypto v0.33.0 // indirect
 	golang.org/x/crypto v0.11.0 // indirect
 	golang.org/x/net v0.12.0 // indirect

src/code.cloudfoundry.org/go.sum

@@ -623,8 +623,6 @@ code.cloudfoundry.org/routing-info v0.0.0-20230612154656-079a27345e39 h1:oNbebj4
 code.cloudfoundry.org/routing-info v0.0.0-20230612154656-079a27345e39/go.mod h1:ykLgqzJGV5PTkvxtfyOy8hcQy7wxPaoV5ZPyk74aqp8=
 code.cloudfoundry.org/runtimeschema v0.0.0-20180622181441-7dcd19348be6 h1:J08p1/LBnhv5BDDf0WLpHRyMJFCws3vd3fLCFL/iVnQ=
 code.cloudfoundry.org/runtimeschema v0.0.0-20180622181441-7dcd19348be6/go.mod h1:QSfnsqiGUdV3OOxqHxQkmUZwycGJNWYdredPgeGuGf4=
-code.cloudfoundry.org/silk v0.0.0-20230728161150-8e48b8d1e681 h1:zSzoPyOnsPVsXtFMtHO6q8x7p/t/RuQBC4qfGJ/2fJ0=
-code.cloudfoundry.org/silk v0.0.0-20230728161150-8e48b8d1e681/go.mod h1:t0kHfcURVs8b0c0IO+U4lQWC+jS7XZwOzugOLcUliQ4=
 code.cloudfoundry.org/tlsconfig v0.0.0-20200131000646-bbe0f8da39b3/go.mod h1:eTbFJpyXRGuFVyg5+oaj9B2eIbIc+0/kZjH8ftbtdew=
 code.cloudfoundry.org/tlsconfig v0.0.0-20230612153104-23c0622de227 h1:QYyb6Ur0Ys6FciDB3+8zCW3eVk7AxAs2++Foa5DAdt0=
 code.cloudfoundry.org/tlsconfig v0.0.0-20230612153104-23c0622de227/go.mod h1:C8SxvGRSutmgzV2FxH8Zwqz2Q8HsaAITQRQFKhlDzPw=

src/code.cloudfoundry.org/silk/.gitignore

@@ -0,0 +1,4 @@
+
+/pkg
+/bin
+bin