From 2464753c021e4bf7562f66f113f3201780e26860 Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 15:38:26 -0700 Subject: [PATCH 01/11] feat: add yaml type for flattening arrays --- src/utils/yaml.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/utils/yaml.js b/src/utils/yaml.js index eaa51e3f..60895b1e 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -183,11 +183,27 @@ export const includeType = new yaml.Type("!include", { }, }); +/** + * !include YAML tag which includes another Yaml file in-place. + * See the tests for example usage. + */ +export const flattenType = new yaml.Type("!flatten", { + kind: "sequence", + construct(data) { + try { + return data.flat(); + } catch (e) { + return data; + } + }, +}); + export const JsYamlTypes = { include: includeType, parameter: parameterType, combine: combineType, esse: esseType, + flatten: flattenType, }; export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ @@ -195,4 +211,5 @@ export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ combineType, esseType, includeType, + flattenType, ]); From 0c84f893e46617d470889b643df800ea35490582 Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 15:43:15 -0700 Subject: [PATCH 02/11] test: basic flatten operation --- tests/enums.js | 1 + tests/fixtures/yaml_flatten_tag.yml | 5 +++++ tests/utils/yaml.flatten.tests.js | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 tests/fixtures/yaml_flatten_tag.yml create mode 100644 tests/utils/yaml.flatten.tests.js diff --git a/tests/enums.js b/tests/enums.js index 83bc91ed..1e727e41 100644 --- a/tests/enums.js +++ b/tests/enums.js @@ -5,3 +5,4 @@ export const YAML_COMBINE_FILE = path.resolve(FIXTURES_DIR, "yaml_combine_tag.ym export const YAML_PARAMETER_FILE = path.resolve(FIXTURES_DIR, "yaml_parameter_tag.yml"); export const YAML_ESSE_FILE = path.resolve(FIXTURES_DIR, "yaml_esse_tag.yml"); export const YAML_INCLUDE_FILE = path.resolve(FIXTURES_DIR, "yaml_include_tag.yml"); +export const YAML_FLATTEN_FILE = path.resolve(FIXTURES_DIR, "yaml_flatten_tag.yml"); diff --git a/tests/fixtures/yaml_flatten_tag.yml b/tests/fixtures/yaml_flatten_tag.yml new file mode 100644 index 00000000..21be0374 --- /dev/null +++ b/tests/fixtures/yaml_flatten_tag.yml @@ -0,0 +1,5 @@ +# convert an array of arrays into a flattened array +case1: !flatten + - [a, b, c] + - d + - [e, f] diff --git a/tests/utils/yaml.flatten.tests.js b/tests/utils/yaml.flatten.tests.js new file mode 100644 index 00000000..01bb851b --- /dev/null +++ b/tests/utils/yaml.flatten.tests.js @@ -0,0 +1,18 @@ +import { expect } from "chai"; +import fs from "fs"; +import yaml from "js-yaml"; + +import { flattenType } from "../../src/utils/yaml"; +import { YAML_FLATTEN_FILE } from "../enums"; + +const flattenSchema = yaml.DEFAULT_SCHEMA.extend([flattenType]); + +describe("YAML tag: !flatten", () => { + const yamlFixture = fs.readFileSync(YAML_FLATTEN_FILE, "utf8"); + + it("should convert an array of arrays into a flattened array", () => { + const parsed = yaml.load(yamlFixture, { schema: flattenSchema }); + const expected = ["a", "b", "c", "d", "e", "f"]; + expect(parsed.case1).to.be.eql(expected); + }); +}); From e72e52ab0f1c4b8b52b66fdc2e094de55a472d8a Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 15:47:17 -0700 Subject: [PATCH 03/11] chore: update package-lock.json --- package-lock.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3687a7ea..f8234b61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1214,8 +1214,9 @@ "dev": true }, "@exabyte-io/esse.js": { - "version": "git+https://github.com/Exabyte-io/esse.git#6f9515988dd2ae589bc74a6096cc743b5dba3225", - "from": "git+https://github.com/Exabyte-io/esse.git#update/SOF-6535", + "version": "2023.7.28-0", + "resolved": "https://registry.npmjs.org/@exabyte-io/esse.js/-/esse.js-2023.7.28-0.tgz", + "integrity": "sha512-B+0LF2GDKO3qb910lOvWS3YpusltJL5KdNqdjO//ljZ677fmH89VgMr82BM8jAhIEx5Uo6bZZ/Fwdsl+oYsUMg==", "requires": { "@babel/cli": "7.16.0", "@babel/core": "7.16.0", From 02c1531aa35cdf9a880b773f3e2953bff9e1a27f Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 15:49:32 -0700 Subject: [PATCH 04/11] docs: update docstring --- src/utils/yaml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/yaml.js b/src/utils/yaml.js index 60895b1e..bd8779b2 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -184,7 +184,7 @@ export const includeType = new yaml.Type("!include", { }); /** - * !include YAML tag which includes another Yaml file in-place. + * !flatten YAML tag for flattening arrays * See the tests for example usage. */ export const flattenType = new yaml.Type("!flatten", { From b3f71f9d6784dc36efb40bd27ffb67880d0db03b Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 16:06:12 -0700 Subject: [PATCH 05/11] feat: add readString Yaml tag --- src/utils/yaml.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/utils/yaml.js b/src/utils/yaml.js index bd8779b2..b2cb5ff0 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -198,12 +198,28 @@ export const flattenType = new yaml.Type("!flatten", { }, }); +/** + * !readString YAML tag for including file contents as a string. + * See the tests for example usage. + */ +export const readStringType = new yaml.Type("!readString", { + kind: "scalar", + construct(data) { + try { + return fs.readFileSync(path.resolve(data), "utf8"); + } catch (e) { + return data; + } + }, +}); + export const JsYamlTypes = { include: includeType, parameter: parameterType, combine: combineType, esse: esseType, flatten: flattenType, + readString: readStringType, }; export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ @@ -212,4 +228,5 @@ export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ esseType, includeType, flattenType, + readStringType, ]); From 4be9773c4b98a794a778b6cbd0cabf0a0e66949f Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 16:06:50 -0700 Subject: [PATCH 06/11] test: reading file contents with readString --- tests/enums.js | 1 + tests/fixtures/test_content.txt | 2 ++ tests/fixtures/yaml_readString_tag.yml | 2 ++ tests/utils/yaml.readString.tests.js | 19 +++++++++++++++++++ 4 files changed, 24 insertions(+) create mode 100644 tests/fixtures/test_content.txt create mode 100644 tests/fixtures/yaml_readString_tag.yml create mode 100644 tests/utils/yaml.readString.tests.js diff --git a/tests/enums.js b/tests/enums.js index 1e727e41..b28eee9e 100644 --- a/tests/enums.js +++ b/tests/enums.js @@ -6,3 +6,4 @@ export const YAML_PARAMETER_FILE = path.resolve(FIXTURES_DIR, "yaml_parameter_ta export const YAML_ESSE_FILE = path.resolve(FIXTURES_DIR, "yaml_esse_tag.yml"); export const YAML_INCLUDE_FILE = path.resolve(FIXTURES_DIR, "yaml_include_tag.yml"); export const YAML_FLATTEN_FILE = path.resolve(FIXTURES_DIR, "yaml_flatten_tag.yml"); +export const YAML_READ_STRING_FILE = path.resolve(FIXTURES_DIR, "yaml_readString_tag.yml"); diff --git a/tests/fixtures/test_content.txt b/tests/fixtures/test_content.txt new file mode 100644 index 00000000..770b738d --- /dev/null +++ b/tests/fixtures/test_content.txt @@ -0,0 +1,2 @@ +Example text +with linebreaks. diff --git a/tests/fixtures/yaml_readString_tag.yml b/tests/fixtures/yaml_readString_tag.yml new file mode 100644 index 00000000..37590ede --- /dev/null +++ b/tests/fixtures/yaml_readString_tag.yml @@ -0,0 +1,2 @@ +# read contents of a file into a string +case1: !readString "tests/fixtures/test_content.txt" diff --git a/tests/utils/yaml.readString.tests.js b/tests/utils/yaml.readString.tests.js new file mode 100644 index 00000000..1f56a727 --- /dev/null +++ b/tests/utils/yaml.readString.tests.js @@ -0,0 +1,19 @@ +import { expect } from "chai"; +import fs from "fs"; +import yaml from "js-yaml"; + +import { readStringType } from "../../src/utils/yaml"; +import { YAML_READ_STRING_FILE } from "../enums"; + +const readStringSchema = yaml.DEFAULT_SCHEMA.extend([readStringType]); + +describe("YAML tag: !readString", () => { + const yamlFixture = fs.readFileSync(YAML_READ_STRING_FILE, "utf8"); + + it("should read contents of a file into a string", () => { + const parsed = yaml.load(yamlFixture, { schema: readStringSchema }); + const expected = "Example text\nwith linebreaks.\n"; + expect(parsed.case1).to.be.a("string"); + expect(parsed.case1).to.be.eql(expected); + }); +}); From e07f89947d17760cfbf40eb52eb05413a64fba6e Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 17:43:29 -0700 Subject: [PATCH 07/11] refactor: rename readString -> readFile --- src/utils/yaml.js | 8 ++++---- tests/enums.js | 2 +- tests/fixtures/yaml_readFile_tag.yml | 2 ++ tests/fixtures/yaml_readString_tag.yml | 2 -- tests/utils/yaml.readFile.tests.js | 19 +++++++++++++++++++ tests/utils/yaml.readString.tests.js | 19 ------------------- 6 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 tests/fixtures/yaml_readFile_tag.yml delete mode 100644 tests/fixtures/yaml_readString_tag.yml create mode 100644 tests/utils/yaml.readFile.tests.js delete mode 100644 tests/utils/yaml.readString.tests.js diff --git a/src/utils/yaml.js b/src/utils/yaml.js index b2cb5ff0..baff9f66 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -199,10 +199,10 @@ export const flattenType = new yaml.Type("!flatten", { }); /** - * !readString YAML tag for including file contents as a string. + * !readFile YAML tag for including file contents as a string. * See the tests for example usage. */ -export const readStringType = new yaml.Type("!readString", { +export const readFileType = new yaml.Type("!readFile", { kind: "scalar", construct(data) { try { @@ -219,7 +219,7 @@ export const JsYamlTypes = { combine: combineType, esse: esseType, flatten: flattenType, - readString: readStringType, + readFile: readFileType, }; export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ @@ -228,5 +228,5 @@ export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ esseType, includeType, flattenType, - readStringType, + readFileType, ]); diff --git a/tests/enums.js b/tests/enums.js index b28eee9e..c2361fa8 100644 --- a/tests/enums.js +++ b/tests/enums.js @@ -6,4 +6,4 @@ export const YAML_PARAMETER_FILE = path.resolve(FIXTURES_DIR, "yaml_parameter_ta export const YAML_ESSE_FILE = path.resolve(FIXTURES_DIR, "yaml_esse_tag.yml"); export const YAML_INCLUDE_FILE = path.resolve(FIXTURES_DIR, "yaml_include_tag.yml"); export const YAML_FLATTEN_FILE = path.resolve(FIXTURES_DIR, "yaml_flatten_tag.yml"); -export const YAML_READ_STRING_FILE = path.resolve(FIXTURES_DIR, "yaml_readString_tag.yml"); +export const YAML_READFILE_FILE = path.resolve(FIXTURES_DIR, "yaml_readFile_tag.yml"); diff --git a/tests/fixtures/yaml_readFile_tag.yml b/tests/fixtures/yaml_readFile_tag.yml new file mode 100644 index 00000000..db69653a --- /dev/null +++ b/tests/fixtures/yaml_readFile_tag.yml @@ -0,0 +1,2 @@ +# read contents of a file into a string +case1: !readFile "tests/fixtures/test_content.txt" diff --git a/tests/fixtures/yaml_readString_tag.yml b/tests/fixtures/yaml_readString_tag.yml deleted file mode 100644 index 37590ede..00000000 --- a/tests/fixtures/yaml_readString_tag.yml +++ /dev/null @@ -1,2 +0,0 @@ -# read contents of a file into a string -case1: !readString "tests/fixtures/test_content.txt" diff --git a/tests/utils/yaml.readFile.tests.js b/tests/utils/yaml.readFile.tests.js new file mode 100644 index 00000000..0d6ed074 --- /dev/null +++ b/tests/utils/yaml.readFile.tests.js @@ -0,0 +1,19 @@ +import { expect } from "chai"; +import fs from "fs"; +import yaml from "js-yaml"; + +import { readFileType } from "../../src/utils/yaml"; +import { YAML_READFILE_FILE } from "../enums"; + +const readFileSchema = yaml.DEFAULT_SCHEMA.extend([readFileType]); + +describe("YAML tag: !readFile", () => { + const yamlFixture = fs.readFileSync(YAML_READFILE_FILE, "utf8"); + + it("should read contents of a file into a string", () => { + const parsed = yaml.load(yamlFixture, { schema: readFileSchema }); + const expected = "Example text\nwith linebreaks.\n"; + expect(parsed.case1).to.be.a("string"); + expect(parsed.case1).to.be.eql(expected); + }); +}); diff --git a/tests/utils/yaml.readString.tests.js b/tests/utils/yaml.readString.tests.js deleted file mode 100644 index 1f56a727..00000000 --- a/tests/utils/yaml.readString.tests.js +++ /dev/null @@ -1,19 +0,0 @@ -import { expect } from "chai"; -import fs from "fs"; -import yaml from "js-yaml"; - -import { readStringType } from "../../src/utils/yaml"; -import { YAML_READ_STRING_FILE } from "../enums"; - -const readStringSchema = yaml.DEFAULT_SCHEMA.extend([readStringType]); - -describe("YAML tag: !readString", () => { - const yamlFixture = fs.readFileSync(YAML_READ_STRING_FILE, "utf8"); - - it("should read contents of a file into a string", () => { - const parsed = yaml.load(yamlFixture, { schema: readStringSchema }); - const expected = "Example text\nwith linebreaks.\n"; - expect(parsed.case1).to.be.a("string"); - expect(parsed.case1).to.be.eql(expected); - }); -}); From f5f0ef7ed7e704489abdd1af1118eb5df173b298 Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 17:43:56 -0700 Subject: [PATCH 08/11] chore: move fixture in before hook --- tests/utils/yaml.combine.tests.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/utils/yaml.combine.tests.js b/tests/utils/yaml.combine.tests.js index 04065f50..e4da8936 100644 --- a/tests/utils/yaml.combine.tests.js +++ b/tests/utils/yaml.combine.tests.js @@ -10,7 +10,12 @@ import { YAML_COMBINE_FILE } from "../enums"; const combineSchema = yaml.DEFAULT_SCHEMA.extend([combineType, esseType]); describe("YAML tag: !combine", () => { - const yamlFixture = fs.readFileSync(YAML_COMBINE_FILE, "utf8"); + let yamlFixture; + + before(() => { + yamlFixture = fs.readFileSync(YAML_COMBINE_FILE, "utf8"); + }); + it("should correctly parse a custom !combine tag with forEach and config keys", () => { const parsed = yaml.load(yamlFixture, { schema: combineSchema }); const expectedResult = [ From 1b3fc7ce0c8497a8d41b8406a44462b54ca7321f Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 18:07:38 -0700 Subject: [PATCH 09/11] feat: add Yaml type for concatenating strings --- src/utils/yaml.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/utils/yaml.js b/src/utils/yaml.js index baff9f66..be3a6c89 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -213,6 +213,24 @@ export const readFileType = new yaml.Type("!readFile", { }, }); +/** + * !join YAML tag for joining strings. + * See the tests for example usage. + */ +export const concatStringType = new yaml.Type("!concatString", { + kind: "sequence", + resolve(data) { + return data.every((d) => lodash.isString(d)); + }, + construct(data) { + try { + return "".concat(...data); + } catch (e) { + return data; + } + }, +}); + export const JsYamlTypes = { include: includeType, parameter: parameterType, @@ -220,6 +238,7 @@ export const JsYamlTypes = { esse: esseType, flatten: flattenType, readFile: readFileType, + concatString: concatStringType, }; export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ @@ -229,4 +248,5 @@ export const JsYamlAllSchemas = yaml.DEFAULT_SCHEMA.extend([ includeType, flattenType, readFileType, + concatStringType, ]); From 1792c6c7fd0676f497166542eb4cfd114824607b Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 18:08:03 -0700 Subject: [PATCH 10/11] test: string concatenation in Yaml --- tests/enums.js | 1 + tests/fixtures/yaml_concatString_tag.yml | 10 ++++++++ tests/utils/yaml.concatString.tests.js | 30 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 tests/fixtures/yaml_concatString_tag.yml create mode 100644 tests/utils/yaml.concatString.tests.js diff --git a/tests/enums.js b/tests/enums.js index c2361fa8..cb343f90 100644 --- a/tests/enums.js +++ b/tests/enums.js @@ -7,3 +7,4 @@ export const YAML_ESSE_FILE = path.resolve(FIXTURES_DIR, "yaml_esse_tag.yml"); export const YAML_INCLUDE_FILE = path.resolve(FIXTURES_DIR, "yaml_include_tag.yml"); export const YAML_FLATTEN_FILE = path.resolve(FIXTURES_DIR, "yaml_flatten_tag.yml"); export const YAML_READFILE_FILE = path.resolve(FIXTURES_DIR, "yaml_readFile_tag.yml"); +export const YAML_CONCAT_STRING_FILE = path.resolve(FIXTURES_DIR, "yaml_concatString_tag.yml"); diff --git a/tests/fixtures/yaml_concatString_tag.yml b/tests/fixtures/yaml_concatString_tag.yml new file mode 100644 index 00000000..7189fbca --- /dev/null +++ b/tests/fixtures/yaml_concatString_tag.yml @@ -0,0 +1,10 @@ +# concatenate three strings +case1: !concatString + - "Hello " + - world + - "!" +# return a string if there is only a single item +case2: !concatString + - Hello +# return an empty string if the array is empty +case3: !concatString [] diff --git a/tests/utils/yaml.concatString.tests.js b/tests/utils/yaml.concatString.tests.js new file mode 100644 index 00000000..df7201c4 --- /dev/null +++ b/tests/utils/yaml.concatString.tests.js @@ -0,0 +1,30 @@ +import { expect } from "chai"; +import fs from "fs"; +import yaml from "js-yaml"; + +import { concatStringType } from "../../src/utils/yaml"; +import { YAML_CONCAT_STRING_FILE } from "../enums"; + +const concatStringSchema = yaml.DEFAULT_SCHEMA.extend([concatStringType]); + +describe("YAML tag: !concatString", () => { + const yamlFixture = fs.readFileSync(YAML_CONCAT_STRING_FILE, "utf8"); + + it("should concatenate two strings", () => { + const parsed = yaml.load(yamlFixture, { schema: concatStringSchema }); + const expected = "Hello world!"; + expect(parsed.case1).to.be.eql(expected); + }); + + it("should return a string if there is only a single item", () => { + const parsed = yaml.load(yamlFixture, { schema: concatStringSchema }); + const expected = "Hello"; + expect(parsed.case2).to.be.eql(expected); + }); + + it("should return an empty string if the array is empty", () => { + const parsed = yaml.load(yamlFixture, { schema: concatStringSchema }); + const expected = ""; + expect(parsed.case3).to.be.eql(expected); + }); +}); From 1a0884db8d778d146c91d31234291221788c5881 Mon Sep 17 00:00:00 2001 From: Alexander Zech Date: Tue, 1 Aug 2023 18:09:25 -0700 Subject: [PATCH 11/11] docs: update docstring --- src/utils/yaml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/yaml.js b/src/utils/yaml.js index be3a6c89..1ea148cb 100644 --- a/src/utils/yaml.js +++ b/src/utils/yaml.js @@ -214,7 +214,7 @@ export const readFileType = new yaml.Type("!readFile", { }); /** - * !join YAML tag for joining strings. + * !concatString YAML tag for concatenating strings. * See the tests for example usage. */ export const concatStringType = new yaml.Type("!concatString", {