From d4adad1c8e6a54d8725aeac45fbe0d752f0f9ac6 Mon Sep 17 00:00:00 2001 From: Yao Sun Date: Tue, 25 Jun 2024 22:16:09 -0700 Subject: [PATCH 1/2] Squashed 'src/YamlCppLib/libyaml/' changes from 2c891fc7..840b65c4 840b65c4 Fix closing flow sequence after explicit key 588eabff Handle closing flow sequence after explicit key abd744ec ci: Install libtool on macOS 1e66c1e1 Fix some typos 51843fe4 Limit depth of nesting by default fb57d89c Update Github actions f8f760f7 ci: Fix build on macOS (#230) acd6f6f0 Add workflow for creating release tarballs git-subtree-dir: src/YamlCppLib/libyaml git-subtree-split: 840b65c40675e2d06bf40405ad3f12dec7f35923 --- .github/workflows/dist.yaml | 28 +++++++++++ .github/workflows/main.yml | 19 +++----- .gitignore | 1 - .makefile | 2 + Makefile.am | 13 ++--- examples/strings.yaml | 2 +- include/yaml.h | 22 +++++++-- pkg/ReadMe.md | 77 ++++++++++++++++++++++++++++++ pkg/docker/.gitignore | 3 ++ pkg/docker/Dockerfile | 32 +++++++++++++ pkg/docker/Makefile | 23 +++++++++ pkg/docker/output/ReadMe | 1 + pkg/docker/scripts/libyaml-dist.sh | 23 +++++++++ src/parser.c | 41 ++++++++++++++++ src/scanner.c | 4 +- tests/run-emitter-test-suite.c | 2 +- tests/run-parser-test-suite.c | 9 +++- tests/run-parser.c | 31 ++++++++++-- tests/test-reader.c | 2 +- 19 files changed, 301 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/dist.yaml create mode 100644 pkg/ReadMe.md create mode 100644 pkg/docker/.gitignore create mode 100644 pkg/docker/Dockerfile create mode 100644 pkg/docker/Makefile create mode 100644 pkg/docker/output/ReadMe create mode 100755 pkg/docker/scripts/libyaml-dist.sh diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml new file mode 100644 index 0000000000..43906d6f10 --- /dev/null +++ b/.github/workflows/dist.yaml @@ -0,0 +1,28 @@ +name: dist + +on: + push: + branches: [ release/* ] + +jobs: + dist: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - run: env | sort + + - name: Get image + run: | + time docker pull yamlio/libyaml-dev + docker images | grep libyaml + + - run: | + make -C pkg/docker libyaml-dist-ci + ls -l pkg/docker/output + + - uses: actions/upload-artifact@v2 + with: + name: release + path: pkg/docker/output/yaml-0* + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c2bdef4710..bd3f87bbec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,8 @@ jobs: build: + env: + CC: ${{ matrix.compiler }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -21,16 +23,15 @@ jobs: - macOS-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: env | sort - name: Install software + if: ${{ matrix.os == 'macOS-latest' }} run: | - if [[ '${{ matrix.os }}' == macOS-latest ]]; then - brew install automake bash coreutils make - echo ::add-path::/usr/local/opt/coreutils/libexec/gnubin - echo ::add-path::/usr/local/opt/make/libexec/gnubin - fi + brew install automake bash coreutils make libtool + echo "/usr/local/opt/coreutils/libexec/gnubin" >> $GITHUB_PATH + echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH - name: Fetch branches run: | git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* @@ -48,12 +49,6 @@ jobs: - name: Compiler version run: ${{ matrix.compiler }} --version - env: - CC: ${{ matrix.compiler }} - run: cmake . - env: - CC: ${{ matrix.compiler }} - run: make - env: - CC: ${{ matrix.compiler }} - run: make test diff --git a/.gitignore b/.gitignore index 90840c8ac2..8257f570c6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ config.h* /configure stamp-h1 !config/config.h.in -/packaging/ /tests/run-dumper /tests/run-emitter /tests/run-emitter-test-suite diff --git a/.makefile b/.makefile index 9d526a4d39..7d6e572390 100644 --- a/.makefile +++ b/.makefile @@ -23,6 +23,8 @@ MAKE_TARGETS := \ all \ all-am \ all-recursive \ + docker-build \ + docker-dist \ install \ test \ test-all \ diff --git a/Makefile.am b/Makefile.am index 551c74071f..45d9c27023 100644 --- a/Makefile.am +++ b/Makefile.am @@ -17,7 +17,7 @@ maintainer-clean-local: -find ${builddir} -name Makefile.in -exec rm -f '{}' ';' distclean-local: - rm -fr tests/run-test-suite packaging + rm -fr tests/run-test-suite -git worktree prune .PHONY: bootstrap @@ -43,12 +43,9 @@ ifeq ($(LIBYAML_TEST_SUITE_RUN_REPO),$(LIBYAML_TEST_SUITE_RUN_REPO_DEFAULT)) git clone --branch $(LIBYAML_TEST_SUITE_RUN_BRANCH) $(LIBYAML_TEST_SUITE_RUN_REPO) $@ endif -packaging: - -git branch --track $@ origin/$@ - git worktree add --force $@ $@ +docker-build: + make -C pkg/docker build -docker-dist: packaging - make -C $ + # Paste the corresponding entry from the Changes file + # Look at an earlier release for how it should look like: + # git show 0.2.3 + git push origin master 0.x.y + +## Create a GitHub release + +Go to "Releases" and click on "Draft a new release". + +Fill in the tag you just created in the previous step. + +Fill in the release title: v0.x.y + +Paste the changelog into the description field. + +Upload the tar.gz and .zip file. + +You can "Save draft" and publish later, or directly click on "Publish release". + +## Update pyyaml.org + +See . + diff --git a/pkg/docker/.gitignore b/pkg/docker/.gitignore new file mode 100644 index 0000000000..f5b0d6de72 --- /dev/null +++ b/pkg/docker/.gitignore @@ -0,0 +1,3 @@ +output/* +!Makefile +!output/ReadMe diff --git a/pkg/docker/Dockerfile b/pkg/docker/Dockerfile new file mode 100644 index 0000000000..71f9df1677 --- /dev/null +++ b/pkg/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +RUN apt-get update \ + && apt-get install -y \ + automake \ + bison \ + build-essential \ + cmake \ + curl \ + doxygen \ + flex \ + git \ + less \ + libtool \ + python \ + vim \ + zip \ + && true + +# http://www.doxygen.nl/manual/install.html + +RUN curl https://sourceforge.net/projects/doxygen/files/rel-1.8.14/doxygen-1.8.14.src.tar.gz/download \ + -L -o /doxygen-1.8.14.src.tar.gz \ + && cd / \ + && tar -xvf doxygen-1.8.14.src.tar.gz \ + && cd doxygen-1.8.14 \ + && mkdir build \ + && cd build \ + && cmake -G "Unix Makefiles" .. \ + && make \ + && make install \ + && true diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile new file mode 100644 index 0000000000..8a53ed30be --- /dev/null +++ b/pkg/docker/Makefile @@ -0,0 +1,23 @@ +DOCKER_IMAGE ?= yamlio/libyaml-dev + +build: + docker build -t $(DOCKER_IMAGE) . + +run: build + docker run -it --rm $(DOCKER_IMAGE) bash + +prepare-git: + rm -rf output/libyaml.git + git clone ../../.git output/libyaml.git + +libyaml-dist: libyaml-dist-ci + +libyaml-dist-ci: prepare-git + docker run --rm -u $$(id -u) \ + -v"$$PWD/output:/output" \ + -v"$$PWD/scripts:/scripts" \ + $(DOCKER_IMAGE) /scripts/libyaml-dist.sh + +clean: + rm -rf output/libyaml.git + rm -rf output/yaml-*.* diff --git a/pkg/docker/output/ReadMe b/pkg/docker/output/ReadMe new file mode 100644 index 0000000000..acdc333151 --- /dev/null +++ b/pkg/docker/output/ReadMe @@ -0,0 +1 @@ +Output directory for build files created by docker diff --git a/pkg/docker/scripts/libyaml-dist.sh b/pkg/docker/scripts/libyaml-dist.sh new file mode 100755 index 0000000000..94e76965ae --- /dev/null +++ b/pkg/docker/scripts/libyaml-dist.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +cp -r /output/libyaml.git /tmp/ +cd /tmp/libyaml.git +./bootstrap +./configure +make dist + +# get the tarball filename +tarballs=(yaml-*.tar.gz) +tarball=${tarballs[0]:?} +dirname=${tarball/.tar.gz/} + +# Copy to output dir +cp "$tarball" /output + +# Create zip archive +cd /tmp +cp "/output/$tarball" . +tar xvf "$tarball" +zip -r "/output/$dirname.zip" "$dirname" diff --git a/src/parser.c b/src/parser.c index ec2f8d3e05..855f1eaae1 100644 --- a/src/parser.c +++ b/src/parser.c @@ -64,6 +64,8 @@ * Public API declarations. */ +int MAX_NESTING_LEVEL = 1000; + YAML_DECLARE(int) yaml_parser_parse(yaml_parser_t *parser, yaml_event_t *event); @@ -80,6 +82,10 @@ yaml_parser_set_parser_error_context(yaml_parser_t *parser, const char *context, yaml_mark_t context_mark, const char *problem, yaml_mark_t problem_mark); +static int +yaml_maximum_level_reached(yaml_parser_t *parser, + yaml_mark_t context_mark, yaml_mark_t problem_mark); + /* * State functions. */ @@ -162,6 +168,12 @@ static int yaml_parser_append_tag_directive(yaml_parser_t *parser, yaml_tag_directive_t value, int allow_duplicates, yaml_mark_t mark); +YAML_DECLARE(void) +yaml_set_max_nest_level(int max) +{ + MAX_NESTING_LEVEL = max; +} + /* * Get the next event. */ @@ -217,6 +229,14 @@ yaml_parser_set_parser_error_context(yaml_parser_t *parser, return 0; } +static int +yaml_maximum_level_reached(yaml_parser_t *parser, + yaml_mark_t context_mark, yaml_mark_t problem_mark) +{ + yaml_parser_set_parser_error_context(parser, + "while parsing", context_mark, "Maximum nesting level reached, set with yaml_set_max_nest_level())", problem_mark); + return 0; +} /* * State dispatcher. @@ -657,6 +677,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (token->type == YAML_FLOW_SEQUENCE_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE; SEQUENCE_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -664,6 +688,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (token->type == YAML_FLOW_MAPPING_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_FLOW_MAPPING_FIRST_KEY_STATE; MAPPING_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -671,6 +699,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (block && token->type == YAML_BLOCK_SEQUENCE_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE; SEQUENCE_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -678,6 +710,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (block && token->type == YAML_BLOCK_MAPPING_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE; MAPPING_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -1022,6 +1058,11 @@ yaml_parser_parse_flow_sequence_entry_mapping_key(yaml_parser_t *parser, return 0; return yaml_parser_parse_node(parser, event, 0, 0); } + else if (token->type == YAML_FLOW_SEQUENCE_END_TOKEN) { + yaml_mark_t mark = token->start_mark; + parser->state = YAML_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE; + return yaml_parser_process_empty_scalar(parser, event, mark); + } else { yaml_mark_t mark = token->end_mark; SKIP_TOKEN(parser); diff --git a/src/scanner.c b/src/scanner.c index c6b4987656..428e16864b 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -273,7 +273,7 @@ * The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation * increase that precedes a block collection (cf. the INDENT token in Python). * The token BLOCK-END denote indentation decrease that ends a block collection - * (cf. the DEDENT token in Python). However YAML has some syntax pecularities + * (cf. the DEDENT token in Python). However YAML has some syntax peculiarities * that makes detections of these tokens more complex. * * The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators @@ -3287,7 +3287,7 @@ yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, /* Check if we are at the end of the scalar. */ - /* Fix for crash unitialized value crash + /* Fix for crash uninitialized value crash * Credit for the bug and input is to OSS Fuzz * Credit for the fix to Alex Gaynor */ diff --git a/tests/run-emitter-test-suite.c b/tests/run-emitter-test-suite.c index ba0f163ad3..950ad5465f 100644 --- a/tests/run-emitter-test-suite.c +++ b/tests/run-emitter-test-suite.c @@ -72,7 +72,7 @@ int main(int argc, char *argv[]) assert(input); if (!yaml_emitter_initialize(&emitter)) { - fprintf(stderr, "Could not initalize the emitter object\n"); + fprintf(stderr, "Could not initialize the emitter object\n"); return 1; } yaml_emitter_set_output_file(&emitter, stdout); diff --git a/tests/run-parser-test-suite.c b/tests/run-parser-test-suite.c index 5bdd66238d..5e29caff82 100644 --- a/tests/run-parser-test-suite.c +++ b/tests/run-parser-test-suite.c @@ -14,9 +14,16 @@ int main(int argc, char *argv[]) int flow = -1; /** default no flow style collections */ int i = 0; int foundfile = 0; + int max_level; + char *output; for (i = 1; i < argc; i++) { - if (strncmp(argv[i], "--flow", 6) == 0) { + if (strncmp(argv[i], "--max-level", 11) == 0) { + i++; + max_level = strtol(argv[i], &output, 10); + yaml_set_max_nest_level(max_level); + } + else if (strncmp(argv[i], "--flow", 6) == 0) { if (i+1 == argc) return usage(1); i++; diff --git a/tests/run-parser.c b/tests/run-parser.c index 13031121ae..9e382504c7 100644 --- a/tests/run-parser.c +++ b/tests/run-parser.c @@ -12,13 +12,31 @@ int main(int argc, char *argv[]) { int number; + int start = 0; + int i = 0; + char *filename; + char *output; + int max_level; + int show_error = 0; if (argc < 2) { printf("Usage: %s file1.yaml ...\n", argv[0]); return 0; } + for (i = 1; i < argc; i++) { + if (strncmp(argv[i], "--max-level", 11) == 0) { + i++; + max_level = strtol(argv[i], &output, 10); + yaml_set_max_nest_level(max_level); + start = i+1; + } + else if (strncmp(argv[i], "--show-error", 12) == 0) { + show_error = 1; + start = i+1; + } + } - for (number = 1; number < argc; number ++) + for (number = start; number < argc; number ++) { FILE *file; yaml_parser_t parser; @@ -27,10 +45,11 @@ main(int argc, char *argv[]) int count = 0; int error = 0; - printf("[%d] Parsing '%s': ", number, argv[number]); + filename = argv[number]; + printf("[%d] Parsing '%s': ", number, filename); fflush(stdout); - file = fopen(argv[number], "rb"); + file = fopen(filename, "rb"); assert(file); assert(yaml_parser_initialize(&parser)); @@ -41,6 +60,12 @@ main(int argc, char *argv[]) { if (!yaml_parser_parse(&parser, &event)) { error = 1; + if (show_error) { + fprintf(stderr, "Parse error: %s\nLine: %lu Column: %lu\n", + parser.problem, + (unsigned long)parser.problem_mark.line + 1, + (unsigned long)parser.problem_mark.column + 1); + } break; } diff --git a/tests/test-reader.c b/tests/test-reader.c index 40f8199e00..5dd3f3e935 100644 --- a/tests/test-reader.c +++ b/tests/test-reader.c @@ -103,7 +103,7 @@ test_case utf8_sequences[] = { test_case boms[] = { - /* {"title", "test!", lenth}, */ + /* {"title", "test!", length}, */ {"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, {"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, From 2b759cc72d1731042f43c8560fbc34a2b6851e9a Mon Sep 17 00:00:00 2001 From: Yao Sun Date: Tue, 25 Jun 2024 22:21:53 -0700 Subject: [PATCH 2/2] Update readme and cgmanifest --- cgmanifest.json | 2 +- src/YamlCppLib/readme.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cgmanifest.json b/cgmanifest.json index 6509cdb8ee..dccbbf2c06 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -15,7 +15,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/yaml/libyaml.git", - "commitHash": "2c891fc7a770e8ba2fec34fc6b545c672beb37e6" + "commitHash": "840b65c40675e2d06bf40405ad3f12dec7f35923" } } }, diff --git a/src/YamlCppLib/readme.md b/src/YamlCppLib/readme.md index 792130c072..2256436502 100644 --- a/src/YamlCppLib/readme.md +++ b/src/YamlCppLib/readme.md @@ -1,6 +1,6 @@ ## YamlCppLib -Do not change code under the libyaml directory; it contains libyaml source code version [0.2.5](https://github.com/yaml/libyaml/releases/tag/0.2.5). +Do not change code under the libyaml directory; it contains libyaml source code version [840b65c](https://github.com/yaml/libyaml/commits/840b65c). It is created using git subtree command: git subtree add --prefix=src/YamlCppLib/libyaml https://github.com/yaml/libyaml.git release/0.2.5 --squash