Skip to content

Commit

Permalink
Check generated wasm file at presubmit (envoyproxy#2621)
Browse files Browse the repository at this point in the history
* wasm file gen

* build and push

* clean up

* add check option

* update prow script mode

* update prow script

* update

* add a make target

* fix

* update wasm gen script

* generate stats plugin wasm

* clean up

* fix log
  • Loading branch information
bianpengyuan authored and istio-testing committed Jan 10, 2020
1 parent 6f7d891 commit becda9d
Show file tree
Hide file tree
Showing 11 changed files with 179 additions and 46 deletions.
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,4 @@ docker/envoy
/artifacts
clang.bazelrc
user.bazelrc
**/plugin.wasm
compile_commands.json
5 changes: 4 additions & 1 deletion Makefile.core.mk
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ build_envoy_asan:
build_wasm:
$(foreach file, $(shell find extensions -name build_wasm.sh), cd $(TOP)/$(shell dirname $(file)) && bash ./build_wasm.sh &&) true

generate_wasm:
./scripts/generate-wasm.sh -b -c

clean:
@bazel clean

Expand Down Expand Up @@ -108,7 +111,7 @@ test_release:
export PATH=$(PATH) CC=$(CC) CXX=$(CXX) BAZEL_BUILD_ARGS="$(BAZEL_BUILD_ARGS)" && ./scripts/release-binary.sh -i

push_release:
export PATH=$(PATH) CC=$(CC) CXX=$(CXX) BAZEL_BUILD_ARGS="$(BAZEL_BUILD_ARGS)" && ./scripts/release-binary.sh -d "$(RELEASE_GCS_PATH)" -p
export PATH=$(PATH) CC=$(CC) CXX=$(CXX) BAZEL_BUILD_ARGS="$(BAZEL_BUILD_ARGS)" && ./scripts/release-binary.sh -d "$(RELEASE_GCS_PATH)" -p && ./scripts/generate-wasm.sh -b -p

.PHONY: build clean test check artifacts

Expand Down
17 changes: 0 additions & 17 deletions extensions/access_log_policy/Makefile

This file was deleted.

2 changes: 1 addition & 1 deletion extensions/metadata_exchange/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ all: plugin.wasm

%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
protoc extensions/common/node_info.proto --cpp_out=.
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -I../../extensions/common -I. -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc','_free'] --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -I../../extensions/common -I. -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
rm -f $*.wast
rm -f extensions/common/node_info.pb.*
chown ${uid}.${gid} $^
Binary file added extensions/metadata_exchange/plugin.wasm
Binary file not shown.
2 changes: 1 addition & 1 deletion extensions/stats/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ all: plugin.wasm
%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
protoc extensions/common/node_info.proto --cpp_out=.
protoc config.proto --cpp_out=.
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} ${CPP_API}/libprotobuf.a -o $*.wasm
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc','_free'] --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} ${CPP_API}/libprotobuf.a -o $*.wasm
rm -f $*.wast
rm -f extensions/common/node_info.pb.* extensions/stats/config.pb.*
chown ${uid}.${gid} $^
Binary file added extensions/stats/plugin.wasm
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright Istio Authors
#
# Copyright 2020 Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -14,7 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
WD=$(dirname $0)
WD=$(cd $WD; pwd)
ROOT=$(dirname $WD)

#######################################
# Presubmit script triggered by Prow. #
#######################################

source "${WD}/proxy-common.inc"

docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work -v $(realpath $PWD/../../extensions):/work/extensions gcr.io/istio-testing/wasmsdk:v2 bash /build_wasm.sh
rmdir extensions
echo 'Generate and check Wasm plugin files'
make generate_wasm
92 changes: 92 additions & 0 deletions scripts/generate-wasm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/bin/bash
#
# Copyright 2020 Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#

set -ex

function usage() {
echo "$0
-b build the wasm sdk image base on ENVOY SHA if it does not exist in `gcr.io/istio-testing` HUB.
If the image already exist in the HUB, this will be noop.
The container will be used to compile wasm files.
-p push the wasm sdk container built from the envoy SHA. Must use with `-b`
-c controls whether to check diff of generated wasm files."
exit 1
}

BUILD_CONTAINER=0
PUSH_CONTAINER=0
CHECK_DIFF=0

while getopts bpc arg ; do
case "${arg}" in
b) BUILD_CONTAINER=1;;
p) PUSH_DOCKER_IMAGE=1;;
c) CHECK_DIFF=1;;
*) usage;;
esac
done

# Get SHA of envoy-wasm repo
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
WORKSPACE=${ROOT}/WORKSPACE
ENVOY_SHA="$(grep -Pom1 "^ENVOY_SHA = \"\K[a-zA-Z0-9]{40}" "${WORKSPACE}")"
IMAGE=gcr.io/istio-testing/wasmsdk
TAG=${ENVOY_SHA}

# Try pull wasm builder image.
docker pull ${IMAGE}:${TAG} || echo "${IMAGE}:${TAG} does not exist"

# If image does not exist, try build it
if [[ "$(docker images -q ${IMAGE}:${TAG} 2> /dev/null)" == "" ]]; then
if [[ ${BUILD_CONTAINER} == 0 ]]; then
echo "no builder image to compile wasm. Add `-b` option to create the builder image"
exit 1
fi
# Clone envoy-wasm repo and checkout to that SHA
TMP_DIR=$(mktemp -d -t envoy-wasm-XXXXXXXXXX)
trap "rm -rf ${TMP_DIR}" EXIT

# Check out to envoy SHA
cd ${TMP_DIR}
git clone https://github.com/envoyproxy/envoy-wasm
cd envoy-wasm
git checkout ${ENVOY_SHA}

# Rebuild and push
cd api/wasm/cpp && docker build -t ${IMAGE}:${TAG} -f Dockerfile-sdk .
if [[ ${PUSH_DOCKER_IMAGE} == 1 ]]; then
docker push ${IMAGE}:${TAG} || "fail to push to gcr.io/istio-testing hub"
fi
fi

# Regenerate all wasm plugins and compare diffs
# Tag image to v2, which is what used by all build wasm script.
docker tag ${IMAGE}:${TAG} ${IMAGE}:v2
cd ${ROOT}
find . -name "*.wasm" -type f -delete
make build_wasm

if [[ ${CHECK_DIFF} == 1 ]]; then
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
echo "wasm files are out of dated and need to be regenerated, run './scripts/generate-wasm.sh -b' to regenerate them"
exit 1
else
echo "wasm files are up to dated"
fi
fi
1 change: 1 addition & 0 deletions test/envoye2e/env/ports.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const (
BasicHTTPGateway
StatsPayload
StatsParallel
StatsWasm

StatsPluginTest

Expand Down
87 changes: 67 additions & 20 deletions test/envoye2e/stats/stats_xds_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,19 @@ filter_chains:
config:
config:
vm_config:
runtime: "envoy.wasm.runtime.null"
runtime: {{ .Vars.WasmRuntime }}
code:
local: { inline_string: "envoy.wasm.metadata_exchange" }
local: { {{ .Vars.MetadataExchangeFilterCode }} }
configuration: "test"
- name: envoy.filters.http.wasm
config:
config:
root_id: "stats_outbound"
vm_config:
vm_id: stats_outbound{{ .N }}
runtime: envoy.wasm.runtime.null
runtime: {{ .Vars.WasmRuntime }}
code:
local: { inline_string: "envoy.wasm.stats" }
local: { {{ .Vars.StatsFilterCode }} }
configuration: |
{ "debug": "false", max_peer_cache_size: 20, field_separator: ";.;" }
- name: envoy.router
Expand Down Expand Up @@ -90,19 +90,19 @@ filter_chains:
config:
config:
vm_config:
runtime: "envoy.wasm.runtime.null"
runtime: {{ .Vars.WasmRuntime }}
code:
local: { inline_string: "envoy.wasm.metadata_exchange" }
local: { {{ .Vars.MetadataExchangeFilterCode }} }
configuration: "test"
- name: envoy.filters.http.wasm
config:
config:
root_id: "stats_inbound"
vm_config:
vm_id: stats_inbound{{ .N }}
runtime: envoy.wasm.runtime.null
runtime: {{ .Vars.WasmRuntime }}
code:
local: { inline_string: "envoy.wasm.stats" }
local: { {{ .Vars.StatsFilterCode }} }
configuration: |
{ "debug": "false", max_peer_cache_size: 20, field_separator: ";.;" }
- name: envoy.router
Expand Down Expand Up @@ -135,12 +135,15 @@ func TestStatsPayload(t *testing.T) {
ports := env.NewPorts(env.StatsPayload)
params := &driver.Params{
Vars: map[string]string{
"ClientPort": fmt.Sprintf("%d", ports.AppToClientProxyPort),
"BackendPort": fmt.Sprintf("%d", ports.BackendPort),
"ClientAdmin": fmt.Sprintf("%d", ports.ClientAdminPort),
"ServerAdmin": fmt.Sprintf("%d", ports.ServerAdminPort),
"ServerPort": fmt.Sprintf("%d", ports.ClientToServerProxyPort),
"RequestCount": "10",
"ClientPort": fmt.Sprintf("%d", ports.AppToClientProxyPort),
"BackendPort": fmt.Sprintf("%d", ports.BackendPort),
"ClientAdmin": fmt.Sprintf("%d", ports.ClientAdminPort),
"ServerAdmin": fmt.Sprintf("%d", ports.ServerAdminPort),
"ServerPort": fmt.Sprintf("%d", ports.ClientToServerProxyPort),
"RequestCount": "10",
"MetadataExchangeFilterCode": "inline_string: \"envoy.wasm.metadata_exchange\"",
"StatsFilterCode": "inline_string: \"envoy.wasm.stats\"",
"WasmRuntime": "envoy.wasm.runtime.null",
},
XDS: int(ports.XDSPort),
}
Expand Down Expand Up @@ -173,12 +176,15 @@ func TestStatsParallel(t *testing.T) {
ports := env.NewPorts(env.StatsParallel)
params := &driver.Params{
Vars: map[string]string{
"ClientPort": fmt.Sprintf("%d", ports.AppToClientProxyPort),
"BackendPort": fmt.Sprintf("%d", ports.BackendPort),
"ClientAdmin": fmt.Sprintf("%d", ports.ClientAdminPort),
"ServerAdmin": fmt.Sprintf("%d", ports.ServerAdminPort),
"ServerPort": fmt.Sprintf("%d", ports.ClientToServerProxyPort),
"RequestCount": "1",
"ClientPort": fmt.Sprintf("%d", ports.AppToClientProxyPort),
"BackendPort": fmt.Sprintf("%d", ports.BackendPort),
"ClientAdmin": fmt.Sprintf("%d", ports.ClientAdminPort),
"ServerAdmin": fmt.Sprintf("%d", ports.ServerAdminPort),
"ServerPort": fmt.Sprintf("%d", ports.ClientToServerProxyPort),
"RequestCount": "1",
"MetadataExchangeFilterCode": "inline_string: \"envoy.wasm.metadata_exchange\"",
"StatsFilterCode": "inline_string: \"envoy.wasm.stats\"",
"WasmRuntime": "envoy.wasm.runtime.null",
},
XDS: int(ports.XDSPort),
}
Expand Down Expand Up @@ -233,3 +239,44 @@ func TestStatsParallel(t *testing.T) {
t.Fatal(err)
}
}

func TestStatsWasm(t *testing.T) {
ports := env.NewPorts(env.StatsWasm)
params := &driver.Params{
Vars: map[string]string{
"ClientPort": fmt.Sprintf("%d", ports.AppToClientProxyPort),
"BackendPort": fmt.Sprintf("%d", ports.BackendPort),
"ClientAdmin": fmt.Sprintf("%d", ports.ClientAdminPort),
"ServerAdmin": fmt.Sprintf("%d", ports.ServerAdminPort),
"ServerPort": fmt.Sprintf("%d", ports.ClientToServerProxyPort),
"RequestCount": "10",
"MetadataExchangeFilterCode": "filename: extensions/metadata_exchange/plugin.wasm",
"StatsFilterCode": "filename: extensions/stats/plugin.wasm",
"WasmRuntime": "envoy.wasm.runtime.v8",
},
XDS: int(ports.XDSPort),
}
params.Vars["ClientMetadata"] = params.LoadTestData("testdata/client_node_metadata.json.tmpl")
params.Vars["ServerMetadata"] = params.LoadTestData("testdata/server_node_metadata.json.tmpl")
params.Vars["StatsConfig"] = params.LoadTestData("testdata/bootstrap/stats.yaml.tmpl")

if err := (&driver.Scenario{
[]driver.Step{
&driver.XDS{},
&driver.Update{Node: "client", Version: "0", Listeners: []string{StatsClientHTTPListener}},
&driver.Update{Node: "server", Version: "0", Listeners: []string{StatsServerHTTPListener}},
&driver.Envoy{Bootstrap: params.LoadTestData("testdata/bootstrap/server.yaml.tmpl")},
&driver.Envoy{Bootstrap: params.LoadTestData("testdata/bootstrap/client.yaml.tmpl")},
&driver.Sleep{1 * time.Second},
&driver.Repeat{N: 10, Step: &driver.Get{ports.AppToClientProxyPort, "hello, world!"}},
&driver.Stats{ports.ClientAdminPort, map[string]driver.StatMatcher{
"istio_requests_total": &driver.ExactStat{"testdata/metric/client_request_total.yaml.tmpl"},
}},
&driver.Stats{ports.ServerAdminPort, map[string]driver.StatMatcher{
"istio_requests_total": &driver.ExactStat{"testdata/metric/server_request_total.yaml.tmpl"},
}},
},
}).Run(params); err != nil {
t.Fatal(err)
}
}

0 comments on commit becda9d

Please sign in to comment.