Skip to content

Commit

Permalink
feat: watch for changed integration test files (#464)
Browse files Browse the repository at this point in the history
  • Loading branch information
boukeversteegh committed Jan 6, 2022
1 parent a6f9131 commit 988cd7e
Show file tree
Hide file tree
Showing 9 changed files with 501 additions and 326 deletions.
4 changes: 3 additions & 1 deletion README.markdown
Original file line number Diff line number Diff line change
Expand Up @@ -404,11 +404,13 @@ The commands below assume you have **Docker** installed. To use a **local** copy
**Workflow**

- Modifying the plugin implementation:
- The most important logic is found in [src/main.ts](src/main.ts).
- Run `yarn bin2ts` or `yarn bin2ts:local`.
_Since the proto files were not changed, you only need to regenerate the typescript files._
- Run `yarn test` to verify the typescript files are compatible with the reference implementation, and pass other tests.
- Updating or adding `.proto` files in the integration directory:
- Run `yarn build:test` to regenerate the integration test files.
- Run `yarn watch` to automatically regenerate test files when proto files change.
- Or run `yarn build:test` to regenerate all integration test files.
- Run `yarn test` to retest.

**Contributing**
Expand Down
33 changes: 20 additions & 13 deletions integration/codegen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
#
# Updates generated output only for the 'simple' and 'value' integration test.
#
# ./codegen.sh simple/simple.bin
#
# Updates generated output for the 'simple' integration test. Used by file watcher.
#
# Each integration test can optionally have a `parameters.txt` file that will
# be used as the ts-proto_opt... args for generating that test's code.

Expand All @@ -25,27 +29,30 @@ N=5

echo "Generating typescript code for integration tests using ${N} cores..."

dir=.
if [ -n "${1}" ]; then
dir="${@}"
cd $INTEGRATION_DIR

if [[ $# -eq 0 ]]; then
FILTER_PATHS=.
else
FILTER_PATHS="${@}"
fi

cd $INTEGRATION_DIR
list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin)
# Finds .bin files that match the given filter paths, which can be directories or paths to the .bin files themselves.
BIN_FILES=$(find $FILTER_PATHS -name "*.bin" -type f | grep -v dump-response.bin)

for file in $list; do
echo "${file}"
for BIN_FILE in $BIN_FILES; do
echo "${BIN_FILE}"
# Strip the longest suffix starting at the 1st slash
dir="${file##./}"
dir="${dir%%/*}"
params=""
TEST_DIR="${BIN_FILE##./}"
TEST_DIR="${TEST_DIR%%/*}"
PARAMS=""

if [ -f "${dir}/parameters.txt" ]; then
params=$(cat "${dir}/parameters.txt")
if [ -f "${TEST_DIR}/parameters.txt" ]; then
PARAMS=$(cat "${TEST_DIR}/parameters.txt")
fi

((i=i%N)); ((i++==0)) && wait
"../node_modules/.bin/ts-node" "./codegen.ts" "${dir}" "${file}" "${params}" &
"../node_modules/.bin/ts-node" "./codegen.ts" "${TEST_DIR}" "${BIN_FILE}" "${PARAMS}" &
done

wait
Binary file modified integration/oneof-properties/oneof.bin
Binary file not shown.
123 changes: 81 additions & 42 deletions integration/pbjs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,59 +4,98 @@
#
# Only a handful of integration tests use these so we hand-code it one-off.
#
# Usage
#
# ./pbjs.sh [TEST_DIRECTORY | PROTO_FILE, ...]
#
# Run for all integration tests:
# ./pbjs.sh
#
# Run for specific integration tests:
# ./pbjs.sh simple value
# ./pbjs.sh simple/simple.proto simple-long/simple.proto

INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))

if [[ $# -eq 0 ]]; then
FILTER_PATHS="$INTEGRATION_DIR"
else
FILTER_PATHS=$(echo "${@}" | xargs realpath)
fi

cd $INTEGRATION_DIR/../
set -e

function match() {
find $FILTER_PATHS -path "$INTEGRATION_DIR/$1/*.proto" -type f | grep -q .
}

# simple/
yarn pbjs --force-message --force-number -t static-module -o integration/simple/pbjs.js integration/simple/simple.proto
yarn pbts --no-comments -o integration/simple/pbjs.d.ts integration/simple/pbjs.js
if match "simple"; then
yarn pbjs --force-message --force-number -t static-module -o integration/simple/pbjs.js integration/simple/simple.proto
yarn pbts --no-comments -o integration/simple/pbjs.d.ts integration/simple/pbjs.js
fi

# simple-long
yarn pbjs --force-message --force-long -t static-module -o integration/simple-long/pbjs.js integration/simple-long/simple.proto
yarn pbts --no-comments -o integration/simple-long/pbjs.d.ts integration/simple-long/pbjs.js
if match "simple-long"; then
yarn run pbjs --force-message --force-long -t static-module -o integration/simple-long/pbjs.js integration/simple-long/simple.proto
yarn run pbts --no-comments -o integration/simple-long/pbjs.d.ts integration/simple-long/pbjs.js
fi

# simple-long-string
yarn pbjs --force-message --force-long -t static-module -o integration/simple-long-string/pbjs.js integration/simple-long-string/simple.proto
yarn pbts --no-comments -o integration/simple-long-string/pbjs.d.ts integration/simple-long-string/pbjs.js
if match "simple-long-string"; then
yarn run pbjs --force-message --force-long -t static-module -o integration/simple-long-string/pbjs.js integration/simple-long-string/simple.proto
yarn run pbts --no-comments -o integration/simple-long-string/pbjs.d.ts integration/simple-long-string/pbjs.js
fi

# vector-tile/
yarn pbjs --force-message --force-number -t static-module -o integration/vector-tile/pbjs.js integration/vector-tile/vector_tile.proto
yarn pbts --no-comments -o integration/vector-tile/pbjs.d.ts integration/vector-tile/pbjs.js
if match "vector-tile"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/vector-tile/pbjs.js integration/vector-tile/vector_tile.proto
yarn run pbts --no-comments -o integration/vector-tile/pbjs.d.ts integration/vector-tile/pbjs.js
fi

# nestjs-metadata/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata/pbjs.js integration/nestjs-metadata/hero.proto
yarn pbts --no-comments -o integration/nestjs-metadata/pbjs.d.ts integration/nestjs-metadata/pbjs.js
if match "nestjs-metadata"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata/pbjs.js integration/nestjs-metadata/hero.proto
yarn run pbts --no-comments -o integration/nestjs-metadata/pbjs.d.ts integration/nestjs-metadata/pbjs.js
fi

# nestjs-metadata-observables/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-observables/pbjs.js integration/nestjs-metadata-observables/hero.proto
yarn pbts --no-comments -o integration/nestjs-metadata-observables/pbjs.d.ts integration/nestjs-metadata-observables/pbjs.js
if match "nestjs-metadata-observables"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-observables/pbjs.js integration/nestjs-metadata-observables/hero.proto
yarn run pbts --no-comments -o integration/nestjs-metadata-observables/pbjs.d.ts integration/nestjs-metadata-observables/pbjs.js
fi

# nestjs-metadata-restparameters/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-restparameters/pbjs.js integration/nestjs-metadata-restparameters/hero.proto
yarn pbts --no-comments -o integration/nestjs-metadata-restparameters/pbjs.d.ts integration/nestjs-metadata-restparameters/pbjs.js
if match "nestjs-metadata-restparameters"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-restparameters/pbjs.js integration/nestjs-metadata-restparameters/hero.proto
yarn run pbts --no-comments -o integration/nestjs-metadata-restparameters/pbjs.d.ts integration/nestjs-metadata-restparameters/pbjs.js
fi

# nestjs-simple/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple/pbjs.js integration/nestjs-simple/hero.proto
yarn pbts --no-comments -o integration/nestjs-simple/pbjs.d.ts integration/nestjs-simple/pbjs.js
if match "nestjs-simple"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple/pbjs.js integration/nestjs-simple/hero.proto
yarn run pbts --no-comments -o integration/nestjs-simple/pbjs.d.ts integration/nestjs-simple/pbjs.js
fi

# nestjs-simple-observables/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-observables/pbjs.js integration/nestjs-simple-observables/hero.proto
yarn pbts --no-comments -o integration/nestjs-simple-observables/pbjs.d.ts integration/nestjs-simple-observables/pbjs.js
if match "nestjs-simple-observables"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-observables/pbjs.js integration/nestjs-simple-observables/hero.proto
yarn run pbts --no-comments -o integration/nestjs-simple-observables/pbjs.d.ts integration/nestjs-simple-observables/pbjs.js
fi

# nestjs-simple-restparameters/
yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-restparameters/pbjs.js integration/nestjs-simple-restparameters/hero.proto
yarn pbts --no-comments -o integration/nestjs-simple-restparameters/pbjs.d.ts integration/nestjs-simple-restparameters/pbjs.js
if match "nestjs-simple-restparameters"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-restparameters/pbjs.js integration/nestjs-simple-restparameters/hero.proto
yarn run pbts --no-comments -o integration/nestjs-simple-restparameters/pbjs.d.ts integration/nestjs-simple-restparameters/pbjs.js
fi

# oneof-proprties/
yarn pbjs --force-message --force-number -t static-module -o integration/oneof-properties/pbjs.js integration/oneof-properties/oneof.proto
yarn pbts --no-comments -o integration/oneof-properties/pbjs.d.ts integration/oneof-properties/pbjs.js
if match "oneof-properties"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/oneof-properties/pbjs.js integration/oneof-properties/oneof.proto
yarn run pbts --no-comments -o integration/oneof-properties/pbjs.d.ts integration/oneof-properties/pbjs.js
fi

# oneof-unions/
yarn pbjs --force-message --force-number -t static-module -o integration/oneof-unions/pbjs.js integration/oneof-unions/oneof.proto
yarn pbts --no-comments -o integration/oneof-unions/pbjs.d.ts integration/oneof-unions/pbjs.js
if match "oneof-unions"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/oneof-unions/pbjs.js integration/oneof-unions/oneof.proto
yarn run pbts --no-comments -o integration/oneof-unions/pbjs.d.ts integration/oneof-unions/pbjs.js
fi

# struct/
yarn pbjs --force-message --force-number -t static-module -o integration/struct/pbjs.js integration/struct/struct.proto
yarn pbts --no-comments -o integration/struct/pbjs.d.ts integration/struct/pbjs.js
if match "struct"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/struct/pbjs.js integration/struct/struct.proto
yarn run pbts --no-comments -o integration/struct/pbjs.d.ts integration/struct/pbjs.js
fi

# value/
yarn pbjs --force-message --force-number -t static-module -o integration/value/pbjs.js integration/value/value.proto
yarn pbts --no-comments -o integration/value/pbjs.d.ts integration/value/pbjs.js
if match "value"; then
yarn run pbjs --force-message --force-number -t static-module -o integration/value/pbjs.js integration/value/value.proto
yarn run pbts --no-comments -o integration/value/pbjs.d.ts integration/value/pbjs.js
fi
6 changes: 5 additions & 1 deletion integration/update-bins.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ else
PLUGIN_PATH="protoc-gen-dump"
fi

PROTO_FILES=$(find . -name "*.proto" -type f)
if [[ $# -eq 0 ]]; then
PROTO_FILES=$(find . -name "*.proto" -type f)
else
PROTO_FILES=$@
fi

for FILE in $PROTO_FILES; do
echo "${FILE}"
Expand Down
86 changes: 86 additions & 0 deletions integration/watch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
const chokidar = require('chokidar');
const spawn = require('child_process').spawn;

main()

function main() {
const yarn = process.platform === 'win32' ? 'yarn.cmd' : 'yarn'

if (process.argv.includes('-h') || process.argv.includes('--help')) {
console.log(`
Watches the integration directory for changes and regenerates .bin and .ts files.
Usage:
$ yarn watch
$ ts-node watch.ts [options]
Options:
-h, --help Show this help message
--polling Use polling instead of native watchers
`)
process.exit(0)
}
const usePolling = process.argv.includes('--polling')

process.chdir("integration");

chokidar
.watch("*/*.proto", { ignoreInitial: true, usePolling })
.on('all', yarnRunHandler(yarn, 'proto2bin'));

chokidar
.watch("*/*.proto", { ignoreInitial: true, usePolling })
.on('all', yarnRunHandler(yarn, 'proto2pbjs'));

chokidar
.watch("*/*.bin", { ignoreInitial: true, usePolling })
.on('all', yarnRunHandler(yarn, 'bin2ts'));
}

const colors = {
none: '',
reset: '\x1b[0m',
green: '\x1b[32m',
red: '\x1b[31m',
yellow: '\x1b[33m',
blue: '\x1b[34m',
magenta: '\x1b[35m',
cyan: '\x1b[36m'
}

function yarnRunHandler(yarn: string, task: string) {
return (event: "add" | "addDir" | "change" | "unlink" | "unlinkDir", path: string) => {
if (event !== 'add' && event !== 'change') {
return;
}

path = path.replace(/\\/g, "/"); // windows

yarnRun(yarn, task, path);
}
}

function yarnRun(yarn: string, task: string, path: string) {
const yarnArgs = [task, path];

console.log(formatLog(colors.green, task, path, `${yarn} ${yarnArgs.join(' ')}`));

const yarnProcess = spawn(yarn, yarnArgs);
yarnProcess.stdout.on('data', (data: Buffer) => console.log(formatLog(colors.none, task, path, data.toString())));
yarnProcess.stderr.on('data', (data: Buffer) => console.error(formatLog(colors.red, task, path, data.toString())));
yarnProcess.on('error', (err: Error) => console.error(formatLog(colors.red, task, path, err.message)));
yarnProcess.on('close', (code: number) => {
if (code !== 0) {
console.error(formatLog(colors.red, task, path, `Exited with code ${code}`));
}
});
}

function formatLog(color: string, task: string, path: string, message: string) {
return message
.split('\n')
.filter(line => line.length)
.map(line => `${colors.reset}${path} ${colors.cyan}[${task}]${colors.reset} ${color}` + line)
.join('\n')
+ colors.reset;
}
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
"test": "yarn jest -c jest.config.js --maxWorkers=2",
"prettier": "prettier --write {src,tests}/**/*.ts",
"prettier:check": "prettier --list-different {src,tests}/**/*.ts",
"setup:docker": "docker-compose build"
"setup:docker": "docker-compose build",
"watch": "ts-node integration/watch.ts"
},
"files": [
"build"
Expand All @@ -45,6 +46,7 @@
"@semantic-release/release-notes-generator": "^9.0.2",
"@types/jest": "^26.0.22",
"@types/node": "^14.14.37",
"chokidar": "^3.5.2",
"jest": "^26.6.3",
"prettier": "^2.2.1",
"reflect-metadata": "^0.1.13",
Expand Down
1 change: 0 additions & 1 deletion src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1009,7 +1009,6 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP
function generateFromJson(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code {
const { options, utils, typeMap } = ctx;
const chunks: Code[] = [];
const choiceFieldChunks: Code[] = [];

// create the basic function declaration
chunks.push(code`
Expand Down

0 comments on commit 988cd7e

Please sign in to comment.