From 25adc4db69e5e00db52c74d8b4412051936d3edf Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 27 Sep 2021 16:13:06 +0300 Subject: [PATCH 01/55] update ajv --- catalog/package-lock.json | 324 +++++++++++++++++++++++++++++++------- catalog/package.json | 2 +- 2 files changed, 265 insertions(+), 61 deletions(-) diff --git a/catalog/package-lock.json b/catalog/package-lock.json index b75e7fb3c3..780575047b 100644 --- a/catalog/package-lock.json +++ b/catalog/package-lock.json @@ -19,7 +19,7 @@ "@urql/devtools": "^2.0.3", "@urql/exchange-graphcache": "^4.1.4", "ace-builds": "^1.4.12", - "ajv": "^6.12.6", + "ajv": "^8.6.3", "aws-sdk": "^2.899.0", "brace": "^0.11.1", "chalk": "^4.1.1", @@ -1281,6 +1281,22 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@eslint/eslintrc/node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1303,6 +1319,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/@graphql-codegen/add": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@graphql-codegen/add/-/add-2.0.2.tgz", @@ -4010,13 +4032,13 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.6.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.3.tgz", + "integrity": "sha512-SMJOdDP6LqTkD0Uq8qLi+gMwSt0imXLSV080qFVwJCpH9U6Mb+SUGHAXM0KNbcBPguytWyvFxcHgMLe2D2XSpw==", "dependencies": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" }, "funding": { @@ -7099,6 +7121,22 @@ "@babel/highlight": "^7.10.4" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -7133,6 +7171,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/espree": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", @@ -7738,6 +7782,28 @@ "yarn": ">=1.0.0" } }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/fork-ts-checker-webpack-plugin/node_modules/schema-utils": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz", @@ -8197,6 +8263,28 @@ "node": ">=6" } }, + "node_modules/har-validator/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/har-validator/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -10273,9 +10361,9 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "node_modules/json-source-map": { "version": "0.6.1", @@ -10366,6 +10454,28 @@ "react": "^16.2.0" } }, + "node_modules/jsoneditor/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/jsoneditor/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "peer": true + }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -14107,7 +14217,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -14331,6 +14440,28 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/schema-utils/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/schema-utils/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/scuid": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/scuid/-/scuid-1.1.0.tgz", @@ -15086,28 +15217,6 @@ "node": ">=10.0.0" } }, - "node_modules/table/node_modules/ajv": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.2.0.tgz", - "integrity": "sha512-WSNGFuyWd//XO8n/m/EaOlNLtO0yL8EXT/74LqT4khdhpZjP7lkj/kT5uwRmGitKEVp/Oj7ZUHeGfPtgHhQ5CA==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/table/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/table/node_modules/slice-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", @@ -17990,6 +18099,18 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -18008,6 +18129,12 @@ "argparse": "^1.0.7", "esprima": "^4.0.0" } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true } } }, @@ -20281,13 +20408,13 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.6.3", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.3.tgz", + "integrity": "sha512-SMJOdDP6LqTkD0Uq8qLi+gMwSt0imXLSV080qFVwJCpH9U6Mb+SUGHAXM0KNbcBPguytWyvFxcHgMLe2D2XSpw==", "requires": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } }, @@ -22350,6 +22477,18 @@ "@babel/highlight": "^7.10.4" } }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -22374,6 +22513,12 @@ "argparse": "^1.0.7", "esprima": "^4.0.0" } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true } } }, @@ -23202,6 +23347,24 @@ "tapable": "^1.0.0" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "schema-utils": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz", @@ -23550,6 +23713,26 @@ "requires": { "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + } } }, "has": { @@ -25130,9 +25313,9 @@ "dev": true }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json-source-map": { "version": "0.6.1", @@ -25200,6 +25383,26 @@ "mobius1-selectr": "^2.4.13", "picomodal": "^3.0.0", "vanilla-picker": "^2.11.2" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "peer": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "peer": true + } } }, "jsoneditor-react": { @@ -28118,8 +28321,7 @@ "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" }, "require-main-filename": { "version": "2.0.0", @@ -28287,6 +28489,26 @@ "@types/json-schema": "^7.0.6", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + } } }, "scuid": { @@ -28885,24 +29107,6 @@ "strip-ansi": "^6.0.0" }, "dependencies": { - "ajv": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.2.0.tgz", - "integrity": "sha512-WSNGFuyWd//XO8n/m/EaOlNLtO0yL8EXT/74LqT4khdhpZjP7lkj/kT5uwRmGitKEVp/Oj7ZUHeGfPtgHhQ5CA==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "slice-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", diff --git a/catalog/package.json b/catalog/package.json index 6c7b650695..aa09458689 100644 --- a/catalog/package.json +++ b/catalog/package.json @@ -63,7 +63,7 @@ "@urql/devtools": "^2.0.3", "@urql/exchange-graphcache": "^4.1.4", "ace-builds": "^1.4.12", - "ajv": "^6.12.6", + "ajv": "^8.6.3", "aws-sdk": "^2.899.0", "brace": "^0.11.1", "chalk": "^4.1.1", From 6b89d60b17a959f660577a699182b02245d7e29f Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 27 Sep 2021 16:13:46 +0300 Subject: [PATCH 02/55] update workflows JSON Schema --- catalog/app/utils/json-schema/json-schema.ts | 26 ++---- shared/schemas/workflows.yml.json | 84 ++------------------ 2 files changed, 15 insertions(+), 95 deletions(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index 9efdf156db..95c4a7fc40 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -1,23 +1,11 @@ -import Ajv from 'ajv' +import Ajv, { SchemaObject, ErrorObject } from 'ajv' import * as dateFns from 'date-fns' import * as R from 'ramda' type CompoundCondition = 'anyOf' | 'oneOf' | 'not' | 'allOf' -export type JsonSchema = Partial< - { - $ref: string - const: string - dateformat: string - default: any - description: string - enum: $TSFixMe[] - format: string - items: JsonSchema - properties: Record - type: string | string[] | JsonSchema[] - } & Record -> +// TODO: use more detailed `Ajv.JSONSchemaType` instead +export type JsonSchema = SchemaObject export const isSchemaArray = (optSchema?: JsonSchema) => optSchema?.type === 'array' @@ -62,7 +50,7 @@ function compoundTypeToHumanString( if (!isSchemaCompound(optSchema)) return '' return optSchema[condition]!.map(schemaTypeToHumanString) - .filter((v) => v !== 'undefined') // NOTE: sic, see default case of `schemaTypeToHumanString` + .filter((v: string) => v !== 'undefined') // NOTE: sic, see default case of `schemaTypeToHumanString` .join(divider) } @@ -100,7 +88,7 @@ function doesTypeMatchCompoundSchema( if (!isSchemaCompound(optSchema)) return false - return optSchema[condition]!.filter(R.has('type')).some((subSchema) => + return optSchema[condition]!.filter(R.has('type')).some((subSchema: JsonSchema) => doesTypeMatchSchema(value, subSchema), ) } @@ -137,12 +125,12 @@ export const EMPTY_SCHEMA = {} export function makeSchemaValidator(optSchema?: JsonSchema) { const schema = optSchema || EMPTY_SCHEMA - const ajv = new Ajv({ useDefaults: true, schemaId: 'auto' }) + const ajv = new Ajv({ useDefaults: true, schemaId: '$id' }) try { const validate = ajv.compile(schema) - return (obj: any): Ajv.ErrorObject[] => { + return (obj: any): ErrorObject[] => { validate(R.clone(obj)) // TODO: add custom errors return validate.errors || [] diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index ba95c7be2d..794726136e 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -1,15 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://quiltdata.com/workflows/config/1", + "$id": "https://schemas.quiltdata.com/workflows-config-1.0.0.json", "type": "object", - "required": [ - "version", - "workflows" - ], + "required": ["version", "workflows"], "additionalProperties": false, "properties": { + "catalog": true, "version": { - "const": "1" + "type": "string", + "pattern": "^[1-9][0-9]*(\\.[0-9]+)?( catalog:[1-9][0-9]*(\\.[0-9]+)?)*$" }, "is_workflow_required": { "type": "boolean", @@ -29,86 +28,19 @@ }, "additionalProperties": { "type": "object", - "required": [ - "name" - ], + "required": ["name"], + "additionalProperties": false, "properties": { + "catalog": true, "name": { "type": "string", "description": "The workflow name displayed by Quilt in the Python API or web UI.", "minLength": 1 }, "description": { - "type": "string", "description": "The workflow description displayed by Quilt in the Python API or web UI.", - "minLength": 1 - }, - "metadata_schema": { - "type": "string", - "description": "JSON Schema $id." - }, - "is_message_required": { - "type": "boolean", - "description": "If true, the user must provide a commit message.", - "default": false - } - } - } - }, - "schemas": { - "type": "object", - "description": "JSON Schemas for validating user-supplied metadata.", - "minProperties": 1, - "additionalProperties": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "url": { - "type": "string", - "description": "URL from where the schema will be obtained.", - "format": "uri" - } - } - } - }, - "successors": { - "type": "object", - "description": "Buckets usable as destination with \"Push to Bucket\" in web UI.", - "examples": [ - { - "s3://bucket1": { - "title": "Bucket 1 Title", - "copy_data": true - }, - "s3://bucket2": { - "title": "Bucket 2 Title", - "copy_data": false - }, - "s3://bucket3": { - "title": "Bucket 3 Title (`copy_data` defaults to `true`)" - } - } - ], - "minProperties": 1, - "propertyNames": { - "format": "uri" - }, - "additionalProperties": { - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { "type": "string", "minLength": 1 - }, - "copy_data": { - "type": "boolean", - "default": true, - "description": "If true, all package entries will be copied to the destination bucket. If false, all entries will remain in their current locations." } } } From 0e14ee97f8aa8c7cb852587f13eda1d0641bf523 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 28 Sep 2021 12:14:41 +0300 Subject: [PATCH 03/55] fix workflows Schema and catalog specific Schema --- shared/schemas/workflows-catalog.yml.json | 63 ++++++++++++++++++++++ shared/schemas/workflows.yml.json | 66 +++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 shared/schemas/workflows-catalog.yml.json diff --git a/shared/schemas/workflows-catalog.yml.json b/shared/schemas/workflows-catalog.yml.json new file mode 100644 index 0000000000..3b2a29c50d --- /dev/null +++ b/shared/schemas/workflows-catalog.yml.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://schemas.quiltdata.com/workflows-config-catalog-1.0.0.json", + "definitions": { + "PackageHandleTemplate": { + "description": "Template for pre-filling package handle, use <%= username %> or <%= directory %> for username and parent directory substitutions", + "type": "string", + "default": "" + }, + "PackageHandleSettings": { + "description": "Map of contexts triggering different templates", + "type": "object", + "additionalProperties": false, + "properties": { + "files": { + "$ref": "#/definitions/PackageHandleTemplate" + }, + "packages": { + "$ref": "#/definitions/PackageHandleTemplate" + } + } + }, + "CatalogSettings": { + "type": "object", + "properties": { + "catalog": { + "type": "object", + "additionalProperties": false, + "properties": { + "package_handle": { + "$ref": "#/definitions/PackageHandleSettings" + } + } + }, + "workflows": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "catalog": { + "type": "object", + "additionalProperties": false, + "properties": { + "package_handle": { + "$ref": "#/definitions/PackageHandleSettings" + } + } + } + } + } + } + } + } + }, + "allOf": [ + { + "$ref": "https://schemas.quiltdata.com/workflows-config-1.0.0.json" + }, + { + "$ref": "#/definitions/CatalogSettings" + } + ] +} diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 794726136e..081e8e4906 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -41,6 +41,72 @@ "description": "The workflow description displayed by Quilt in the Python API or web UI.", "type": "string", "minLength": 1 + }, + "metadata_schema": { + "type": "string", + "description": "JSON Schema $id." + }, + "is_message_required": { + "type": "boolean", + "description": "If true, the user must provide a commit message.", + "default": false + }, + "object_schema": { + "type": "string" + } + } + } + }, + "schemas": { + "type": "object", + "description": "JSON Schemas for validating user-supplied metadata.", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "required": ["url"], + "properties": { + "url": { + "type": "string", + "description": "URL from where the schema will be obtained.", + "format": "uri" + } + } + } + }, + "successors": { + "type": "object", + "description": "Buckets usable as destination with \"Push to Bucket\" in web UI.", + "examples": [ + { + "s3://bucket1": { + "title": "Bucket 1 Title", + "copy_data": true + }, + "s3://bucket2": { + "title": "Bucket 2 Title", + "copy_data": false + }, + "s3://bucket3": { + "title": "Bucket 3 Title (`copy_data` defaults to `true`)" + } + } + ], + "minProperties": 1, + "propertyNames": { + "format": "uri" + }, + "additionalProperties": { + "type": "object", + "required": ["title"], + "properties": { + "title": { + "type": "string", + "minLength": 1 + }, + "copy_data": { + "type": "boolean", + "default": true, + "description": "If true, all package entries will be copied to the destination bucket. If false, all entries will remain in their current locations." } } } From 773c5ef95d11fbae0fb7d60f69524a1d1187b5fa Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 28 Sep 2021 12:46:41 +0300 Subject: [PATCH 04/55] apply new Schema validation for catalog workflows --- catalog/app/utils/json-schema/json-schema.ts | 36 +++++++++++++++----- catalog/app/utils/workflows.ts | 5 ++- catalog/package-lock.json | 25 ++++++++++++++ catalog/package.json | 1 + 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index 95c4a7fc40..1cee104770 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -1,4 +1,5 @@ -import Ajv, { SchemaObject, ErrorObject } from 'ajv' +import Ajv, { SchemaObject, ErrorObject, Options } from 'ajv' +import addFormats from 'ajv-formats' import * as dateFns from 'date-fns' import * as R from 'ramda' @@ -120,20 +121,37 @@ export function doesTypeMatchSchema(value: any, optSchema?: JsonSchema): boolean ])(optSchema) } -export const EMPTY_SCHEMA = {} +export const EMPTY_SCHEMA: JsonSchema = {} -export function makeSchemaValidator(optSchema?: JsonSchema) { - const schema = optSchema || EMPTY_SCHEMA - - const ajv = new Ajv({ useDefaults: true, schemaId: '$id' }) +export function makeSchemaValidator(optSchema?: JsonSchema, optSchemas?: JsonSchema[]) { + let mainSchema = R.clone(optSchema || EMPTY_SCHEMA) + if (!mainSchema.$id) { + // Make further code more universal by using one format: `id` → `$id` + if (mainSchema.id) { + mainSchema = R.pipe(R.assoc('$id', mainSchema.id), R.dissoc('id'))(mainSchema) + } else { + mainSchema = R.assoc('$id', 'main_schema', mainSchema) + } + } + const schemas = optSchemas ? [mainSchema, ...optSchemas] : [mainSchema] + + const { $id } = schemas[0] + const options: Options = { + allErrors: true, + schemaId: '$id', + schemas, + useDefaults: true, + } + const ajv = new Ajv(options) + addFormats(ajv, ['uri']) try { - const validate = ajv.compile(schema) + if (!$id) return () => [new Error('$id is not provided')] return (obj: any): ErrorObject[] => { - validate(R.clone(obj)) + ajv.validate($id, R.clone(obj)) // TODO: add custom errors - return validate.errors || [] + return ajv.errors || [] } } catch (e) { // TODO: add custom errors diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index 0b679a5adc..ed60fe4a07 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -4,6 +4,7 @@ import { makeSchemaValidator } from 'utils/json-schema' import * as s3paths from 'utils/s3paths' import yaml from 'utils/yaml' import workflowsConfigSchema from 'schemas/workflows.yml.json' +import workflowsCatalogConfigSchema from 'schemas/workflows-catalog.yml.json' import * as bucketErrors from 'containers/Bucket/errors' interface WorkflowsYaml { @@ -106,7 +107,9 @@ const parseSuccessor = (url: string, successor: SuccessorYaml): Successor => ({ url, }) -const workflowsConfigValidator = makeSchemaValidator(workflowsConfigSchema) +const workflowsConfigValidator = makeSchemaValidator(workflowsCatalogConfigSchema, [ + workflowsConfigSchema, +]) function validateConfig(data: unknown): asserts data is WorkflowsYaml { const errors = workflowsConfigValidator(data) diff --git a/catalog/package-lock.json b/catalog/package-lock.json index 780575047b..4490d016f0 100644 --- a/catalog/package-lock.json +++ b/catalog/package-lock.json @@ -20,6 +20,7 @@ "@urql/exchange-graphcache": "^4.1.4", "ace-builds": "^1.4.12", "ajv": "^8.6.3", + "ajv-formats": "^2.1.1", "aws-sdk": "^2.899.0", "brace": "^0.11.1", "chalk": "^4.1.1", @@ -4046,6 +4047,22 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", @@ -20418,6 +20435,14 @@ "uri-js": "^4.2.2" } }, + "ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "requires": { + "ajv": "^8.0.0" + } + }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", diff --git a/catalog/package.json b/catalog/package.json index aa09458689..d439850255 100644 --- a/catalog/package.json +++ b/catalog/package.json @@ -64,6 +64,7 @@ "@urql/exchange-graphcache": "^4.1.4", "ace-builds": "^1.4.12", "ajv": "^8.6.3", + "ajv-formats": "^2.1.1", "aws-sdk": "^2.899.0", "brace": "^0.11.1", "chalk": "^4.1.1", From e03290882516afcc52175bdc536f8accc8c479aa Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 28 Sep 2021 12:50:22 +0300 Subject: [PATCH 05/55] fix Schema format --- catalog/config-schema.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/catalog/config-schema.json b/catalog/config-schema.json index a7f6724516..8a745b7b78 100644 --- a/catalog/config-schema.json +++ b/catalog/config-schema.json @@ -97,6 +97,10 @@ "ssoProviders": { "type": "string", "description": "Space-separated list of SSO providers." + }, + "build_version": { + "type": "string", + "description": "Optional" } }, "required": [ @@ -112,11 +116,6 @@ "ssoAuth", "ssoProviders" ], - "additionalProperties": { - "build_version": { - "type": "string" - } - }, "definitions": { "Url": { "type": "string", From 83cf642c4ddd588a639ade9e54e25eaf539c65c2 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 28 Sep 2021 12:54:32 +0300 Subject: [PATCH 06/55] this feature is not ready yet --- shared/schemas/workflows.yml.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 081e8e4906..947c277c0f 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -50,9 +50,6 @@ "type": "boolean", "description": "If true, the user must provide a commit message.", "default": false - }, - "object_schema": { - "type": "string" } } } From 49b4a4b24ec325a21307888a6283ac16b289649e Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 10:09:02 +0300 Subject: [PATCH 07/55] fix tests according to ajv update --- .../PackageDialog/PackageDialog.spec.ts | 2 +- .../app/utils/json-schema/json-schema.spec.ts | 101 ++++++++++++------ .../app/utils/json-schema/mocks/compound.js | 4 +- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.spec.ts b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.spec.ts index c40ec58688..0a14f240a2 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.spec.ts +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.spec.ts @@ -55,7 +55,7 @@ describe('containers/Bucket/PackageDialog/PackageDialog', () => { test("should return error when metadata isn't compliant with Schema", () => { expect(PD.mkMetaValidator({ type: 'array' })({ any: 'thing' })).toMatchObject([ - { message: 'should be array' }, + { message: 'must be array' }, ]) }) diff --git a/catalog/app/utils/json-schema/json-schema.spec.ts b/catalog/app/utils/json-schema/json-schema.spec.ts index 7084cc0e82..0d4a0e242d 100644 --- a/catalog/app/utils/json-schema/json-schema.spec.ts +++ b/catalog/app/utils/json-schema/json-schema.spec.ts @@ -21,12 +21,9 @@ describe('utils/json-schema', () => { }) it('should throw an error when schema is incorrect', () => { - const validate = makeSchemaValidator(incorrect.schema) - const errors = validate({}) - - expect(errors).toHaveLength(1) - expect(errors[0]).toBeInstanceOf(Error) - expect(errors[0].message).toMatch(/schema is invalid/) + expect(() => { + makeSchemaValidator(incorrect.schema) + }).toThrowError(/schema is invalid/) }) describe('validator for Schema with enum types', () => { @@ -36,7 +33,7 @@ describe('utils/json-schema', () => { const invalid = { a: 123, b: 'value', optEnum: 'value' } const errors = validate(invalid) expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ dataPath: '.optEnum', keyword: 'enum' }) + expect(errors[0]).toMatchObject({ instancePath: '/optEnum', keyword: 'enum' }) }) it("shouldn't return error, when value matches enum", () => { @@ -53,7 +50,7 @@ describe('utils/json-schema', () => { const errors = validate(invalid) expect(errors).toHaveLength(1) expect(errors[0]).toMatchObject({ - dataPath: '.longNestedList[0]', + instancePath: '/longNestedList/0', keyword: 'type', }) }) @@ -63,7 +60,7 @@ describe('utils/json-schema', () => { const errors = validate(invalid) expect(errors).toHaveLength(1) expect(errors[0]).toMatchObject({ - dataPath: '.longNestedList[0][0][0][0][0][0][0][0][0][0]', + instancePath: '/longNestedList/0/0/0/0/0/0/0/0/0/0', keyword: 'type', }) }) @@ -73,13 +70,13 @@ describe('utils/json-schema', () => { const errors = validate(invalid) expect(errors).toHaveLength(1) expect(errors[0]).toMatchObject({ - dataPath: '.longNestedList[0][0][0][0][0][0][0][0][0][4]', + instancePath: '/longNestedList/0/0/0/0/0/0/0/0/0/4', keyword: 'type', }) }) it("shouldn't return error, when value matches array structure and types inside last nesting level", () => { - const valid = { longNestedList: [[[[[[[[[[1, 2.5, 3, 10e100, Infinity]]]]]]]]]] } + const valid = { longNestedList: [[[[[[[[[[1, 2.5, 3, 10e100]]]]]]]]]] } expect(validate(valid)).toHaveLength(0) }) }) @@ -91,14 +88,14 @@ describe('utils/json-schema', () => { const invalidNumber = { a: 'b', b: 'b' } const errorsNumber = validate(invalidNumber) expect(errorsNumber).toHaveLength(1) - expect(errorsNumber[0]).toMatchObject({ dataPath: '.a', keyword: 'type' }) + expect(errorsNumber[0]).toMatchObject({ instancePath: '/a', keyword: 'type' }) }) it('should return error, when string is expected but number is provided', () => { const invalidString = { a: 123, b: 123 } const errorsString = validate(invalidString) expect(errorsString).toHaveLength(1) - expect(errorsString[0]).toMatchObject({ dataPath: '.b', keyword: 'type' }) + expect(errorsString[0]).toMatchObject({ instancePath: '/b', keyword: 'type' }) }) it("shouldn't return error, when object matches Schema types", () => { @@ -114,21 +111,34 @@ describe('utils/json-schema', () => { const invalidNull = { nullValue: 123 } const errorsNull = validate(invalidNull) expect(errorsNull).toHaveLength(1) - expect(errorsNull[0]).toMatchObject({ dataPath: '.nullValue', keyword: 'type' }) + expect(errorsNull[0]).toMatchObject({ + instancePath: '/nullValue', + keyword: 'type', + }) }) it("should return error, when value doesn't match boolean", () => { const invalidBool = { boolValue: 0 } const errorsBool = validate(invalidBool) expect(errorsBool).toHaveLength(1) - expect(errorsBool[0]).toMatchObject({ dataPath: '.boolValue', keyword: 'type' }) + expect(errorsBool[0]).toMatchObject({ + instancePath: '/boolValue', + keyword: 'type', + }) }) it("should return error, when value doesn't match boolean as enum", () => { const invalidEnum = { enumBool: null } const errorsEnum = validate(invalidEnum) - expect(errorsEnum).toHaveLength(1) - expect(errorsEnum[0]).toMatchObject({ dataPath: '.enumBool', keyword: 'type' }) + expect(errorsEnum).toHaveLength(2) + expect(errorsEnum[0]).toMatchObject({ + instancePath: '/enumBool', + keyword: 'type', + }) + expect(errorsEnum[1]).toMatchObject({ + instancePath: '/enumBool', + keyword: 'enum', + }) }) it("shouldn't return error, when object matches Schema types", () => { @@ -156,15 +166,15 @@ describe('utils/json-schema', () => { expect(errorsAnyOf).toHaveLength(3) expect(errorsAnyOf).toMatchObject([ { - dataPath: '.numOrString', + instancePath: '/numOrString', keyword: 'type', }, { - dataPath: '.numOrString', + instancePath: '/numOrString', keyword: 'type', }, { - dataPath: '.numOrString', + instancePath: '/numOrString', keyword: 'anyOf', }, ]) @@ -176,15 +186,15 @@ describe('utils/json-schema', () => { expect(errorsOneOf).toHaveLength(3) expect(errorsOneOf).toMatchObject([ { - dataPath: '.intOrNonNumberOrLess3', + instancePath: '/intOrNonNumberOrLess3', keyword: 'maximum', }, { - dataPath: '.intOrNonNumberOrLess3', + instancePath: '/intOrNonNumberOrLess3', keyword: 'type', }, { - dataPath: '.intOrNonNumberOrLess3', + instancePath: '/intOrNonNumberOrLess3', keyword: 'oneOf', }, ]) @@ -193,9 +203,13 @@ describe('utils/json-schema', () => { it("shouldn't return error, when value doesn't match `allOf` type", () => { const invalidAllOf = { intLessThan3: 'a' } const errorsAllOf = validate(invalidAllOf) - expect(errorsAllOf).toHaveLength(1) + expect(errorsAllOf).toHaveLength(2) expect(errorsAllOf[0]).toMatchObject({ - dataPath: '.intLessThan3', + instancePath: '/intLessThan3', + keyword: 'type', + }) + expect(errorsAllOf[1]).toMatchObject({ + instancePath: '/intLessThan3', keyword: 'type', }) }) @@ -203,11 +217,15 @@ describe('utils/json-schema', () => { it("shouldn't return error, when value doesn't match `allOf` comparing condition", () => { const invalidAllOf = { intLessThan3: 3.5 } const errorsAllOf = validate(invalidAllOf) - expect(errorsAllOf).toHaveLength(1) + expect(errorsAllOf).toHaveLength(2) expect(errorsAllOf[0]).toMatchObject({ - dataPath: '.intLessThan3', + instancePath: '/intLessThan3', keyword: 'maximum', }) + expect(errorsAllOf[1]).toMatchObject({ + instancePath: '/intLessThan3', + keyword: 'type', + }) }) it("shouldn't return error, when object matches Schema types", () => { @@ -218,14 +236,19 @@ describe('utils/json-schema', () => { }) describe('validator for Schema with types as array', () => { + // TODO: use allowUnionTypes const validate = makeSchemaValidator(compound.schemaTypeArray) it("should return error, when root property doesn't match type", () => { const invalidProperty = { strOrNum: [], strOrNumList: [{}, null, true] } const errorsProperty = validate(invalidProperty) - expect(errorsProperty).toHaveLength(1) + expect(errorsProperty).toHaveLength(4) expect(errorsProperty[0]).toMatchObject({ - dataPath: '.strOrNum', + instancePath: '/strOrNum', + keyword: 'type', + }) + expect(errorsProperty[1]).toMatchObject({ + instancePath: '/strOrNumList/0', keyword: 'type', }) }) @@ -233,9 +256,17 @@ describe('utils/json-schema', () => { it("should return error, when first element inside array doesn't match type", () => { const invalidItem = { strOrNum: 123, strOrNumList: [{}, null, true] } const errorsItem = validate(invalidItem) - expect(errorsItem).toHaveLength(1) + expect(errorsItem).toHaveLength(3) expect(errorsItem[0]).toMatchObject({ - dataPath: '.strOrNumList[0]', + instancePath: '/strOrNumList/0', + keyword: 'type', + }) + expect(errorsItem[1]).toMatchObject({ + instancePath: '/strOrNumList/1', + keyword: 'type', + }) + expect(errorsItem[2]).toMatchObject({ + instancePath: '/strOrNumList/2', keyword: 'type', }) }) @@ -243,9 +274,13 @@ describe('utils/json-schema', () => { it("should return error, when second element inside array doesn't match type", () => { const invalidItem = { strOrNum: 123, strOrNumList: [123, null, true] } const errorsItem = validate(invalidItem) - expect(errorsItem).toHaveLength(1) + expect(errorsItem).toHaveLength(2) expect(errorsItem[0]).toMatchObject({ - dataPath: '.strOrNumList[1]', + instancePath: '/strOrNumList/1', + keyword: 'type', + }) + expect(errorsItem[1]).toMatchObject({ + instancePath: '/strOrNumList/2', keyword: 'type', }) }) diff --git a/catalog/app/utils/json-schema/mocks/compound.js b/catalog/app/utils/json-schema/mocks/compound.js index c2e6a1ef82..245369f94b 100644 --- a/catalog/app/utils/json-schema/mocks/compound.js +++ b/catalog/app/utils/json-schema/mocks/compound.js @@ -13,10 +13,10 @@ export const schemaAnyOf = { ], }, intOrNonNumberOrLess3: { - oneOf: [{ maximum: 3 }, { type: 'integer' }], + oneOf: [{ maximum: 3, type: 'number' }, { type: 'integer' }], }, intLessThan3: { - allOf: [{ maximum: 3 }, { type: 'integer' }], + allOf: [{ maximum: 3, type: 'number' }, { type: 'integer' }], }, }, } From 7254f02b07f0722329761c40801af225a0ef8a43 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 10:15:29 +0300 Subject: [PATCH 08/55] rewrite ajv options --- catalog/app/utils/json-schema/json-schema.spec.ts | 5 +++-- catalog/app/utils/json-schema/json-schema.ts | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/catalog/app/utils/json-schema/json-schema.spec.ts b/catalog/app/utils/json-schema/json-schema.spec.ts index 0d4a0e242d..ce4f55cc42 100644 --- a/catalog/app/utils/json-schema/json-schema.spec.ts +++ b/catalog/app/utils/json-schema/json-schema.spec.ts @@ -236,8 +236,9 @@ describe('utils/json-schema', () => { }) describe('validator for Schema with types as array', () => { - // TODO: use allowUnionTypes - const validate = makeSchemaValidator(compound.schemaTypeArray) + const validate = makeSchemaValidator(compound.schemaTypeArray, [], { + allowUnionTypes: true, + }) it("should return error, when root property doesn't match type", () => { const invalidProperty = { strOrNum: [], strOrNumList: [{}, null, true] } diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index 1cee104770..d992029e1b 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -123,7 +123,11 @@ export function doesTypeMatchSchema(value: any, optSchema?: JsonSchema): boolean export const EMPTY_SCHEMA: JsonSchema = {} -export function makeSchemaValidator(optSchema?: JsonSchema, optSchemas?: JsonSchema[]) { +export function makeSchemaValidator( + optSchema?: JsonSchema, + optSchemas?: JsonSchema[], + ajvOptions?: Options, +) { let mainSchema = R.clone(optSchema || EMPTY_SCHEMA) if (!mainSchema.$id) { // Make further code more universal by using one format: `id` → `$id` @@ -141,6 +145,7 @@ export function makeSchemaValidator(optSchema?: JsonSchema, optSchemas?: JsonSch schemaId: '$id', schemas, useDefaults: true, + ...ajvOptions, } const ajv = new Ajv(options) addFormats(ajv, ['uri']) From 914ba3860d94b0d02cc48bb98b6f315d0e456c38 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 10:25:35 +0300 Subject: [PATCH 09/55] fix ajv errors format --- .../Bucket/PackageDialog/MetaInputErrorHelper.js | 6 +++--- catalog/app/containers/Bucket/errors.tsx | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/MetaInputErrorHelper.js b/catalog/app/containers/Bucket/PackageDialog/MetaInputErrorHelper.js index 2b79840b91..5695a6bc80 100644 --- a/catalog/app/containers/Bucket/PackageDialog/MetaInputErrorHelper.js +++ b/catalog/app/containers/Bucket/PackageDialog/MetaInputErrorHelper.js @@ -18,9 +18,9 @@ function SingleError({ error }) { return ( - {error.dataPath && ( + {error.instancePath && ( <> - {error.dataPath} + {error.instancePath} )} {error.message} @@ -34,7 +34,7 @@ export default function ErrorHelper({ className, error }) { return (
{Array.isArray(error) ? ( - error.map((e) => ) + error.map((e) => ) ) : ( )} diff --git a/catalog/app/containers/Bucket/errors.tsx b/catalog/app/containers/Bucket/errors.tsx index fd4ad9d7cf..54b18c50e8 100644 --- a/catalog/app/containers/Bucket/errors.tsx +++ b/catalog/app/containers/Bucket/errors.tsx @@ -43,7 +43,7 @@ export class FileNotFound extends BucketError {} export class VersionNotFound extends BucketError {} export interface BucketPreferencesInvalidProps { - errors: { dataPath?: string; message?: string }[] + errors: { instancePath?: string; message?: string }[] } export class BucketPreferencesInvalid extends BucketError { @@ -51,14 +51,16 @@ export class BucketPreferencesInvalid extends BucketError { constructor(props: BucketPreferencesInvalidProps) { super( - props.errors.map(({ dataPath, message }) => `${dataPath} ${message}`).join(', '), + props.errors + .map(({ instancePath, message }) => `${instancePath} ${message}`) + .join(', '), props, ) } } export interface WorkflowsConfigInvalidProps { - errors: { dataPath?: string; message?: string }[] + errors: { instancePath?: string; message?: string }[] } export class WorkflowsConfigInvalid extends BucketError { @@ -66,7 +68,9 @@ export class WorkflowsConfigInvalid extends BucketError { constructor(props: WorkflowsConfigInvalidProps) { super( - props.errors.map(({ dataPath, message }) => `${dataPath} ${message}`).join(', '), + props.errors + .map(({ instancePath, message }) => `${instancePath} ${message}`) + .join(', '), props, ) } From b778edbc3136bee575b8768ee911f2e6dde2fb55 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 15:01:09 +0300 Subject: [PATCH 10/55] fix schema --- shared/schemas/quilt_summarize.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/schemas/quilt_summarize.json b/shared/schemas/quilt_summarize.json index 0aea990f63..f909adec3f 100644 --- a/shared/schemas/quilt_summarize.json +++ b/shared/schemas/quilt_summarize.json @@ -3,7 +3,7 @@ "fileShortcut": { "type": "string", "description": "Path relative to quilt_summarize.json", - "example": "file1.json" + "examples": ["file1.json"] }, "fileExtended": { @@ -47,12 +47,12 @@ "anyOf": [ { "description": "Ratio number for flex-based width", - "example": 1.5, + "examples": [1.5], "type": "number" }, { "description": "Width in pixels or percents", - "example": "100px", + "examples": ["100px"], "type": "string" } ] From 43d94bd557dd2ed38aa8351fa09b868d3ae30dd8 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 15:09:09 +0300 Subject: [PATCH 11/55] add package_handle support --- catalog/app/utils/packageHandle.ts | 26 +++++++++++++++++++ catalog/app/utils/workflows.ts | 40 ++++++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/catalog/app/utils/packageHandle.ts b/catalog/app/utils/packageHandle.ts index 8a38cae45d..c0c4c8b1f4 100644 --- a/catalog/app/utils/packageHandle.ts +++ b/catalog/app/utils/packageHandle.ts @@ -1,3 +1,4 @@ +import lodashTemplate from 'lodash/template' import * as R from 'ramda' export interface PackageHandle { @@ -9,3 +10,28 @@ export interface PackageHandle { export function shortenRevision(fullRevision: string): string { return R.take(10, fullRevision) } + +type Context = 'files' | 'packages' + +export type NameTemplates = Partial> + +type Options = { + username?: string + directory?: string +} + +export function execTemplateItem(template: string, options?: Options): string | null { + try { + return lodashTemplate(template)(options) + } catch (error) { + return null + } +} + +export function execTemplate( + templatesDict: NameTemplates, + context: Context, + options?: Options, +): string { + return execTemplateItem(templatesDict[context] || '', options) || '' +} diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index ed60fe4a07..6f0eb38267 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -1,6 +1,7 @@ import * as R from 'ramda' import { makeSchemaValidator } from 'utils/json-schema' +import type * as packageHandleUtils from 'utils/packageHandle' import * as s3paths from 'utils/s3paths' import yaml from 'utils/yaml' import workflowsConfigSchema from 'schemas/workflows.yml.json' @@ -8,19 +9,25 @@ import workflowsCatalogConfigSchema from 'schemas/workflows-catalog.yml.json' import * as bucketErrors from 'containers/Bucket/errors' interface WorkflowsYaml { - version: '1' - is_workflow_required?: boolean default_workflow?: string - workflows: Record + is_workflow_required?: boolean + catalog: { + package_handle?: packageHandleUtils.NameTemplates + } schemas?: Record successors?: Record + version: '1' + workflows: Record } interface WorkflowYaml { - name: string description?: string - metadata_schema?: string is_message_required?: boolean + metadata_schema?: string + name: string + catalog: { + package_handle?: packageHandleUtils.NameTemplates + } } interface SuccessorYaml { @@ -44,24 +51,41 @@ export interface Workflow { isDefault: boolean isDisabled: boolean name?: string + packageName: Required schema?: Schema slug: string | typeof notAvailable | typeof notSelected } export interface WorkflowsConfig { isWorkflowRequired: boolean + packageName: Required successors: Successor[] workflows: Workflow[] } +const defaultPackageNameTemplates = { + files: '', + packages: '', +} + export const notAvailable = Symbol('not available') +const parsePackageNameTemplates = ( + globalTemplates?: packageHandleUtils.NameTemplates, + workflowTemplates?: packageHandleUtils.NameTemplates, +): Required => ({ + ...defaultPackageNameTemplates, + ...globalTemplates, + ...workflowTemplates, +}) + export const notSelected = Symbol('not selected') function getNoWorkflow(data: WorkflowsYaml, hasConfig: boolean): Workflow { return { isDefault: !data.default_workflow, isDisabled: data.is_workflow_required !== false, + packageName: parsePackageNameTemplates(data.catalog?.package_handle), slug: hasConfig ? notSelected : notAvailable, } } @@ -70,6 +94,7 @@ const COPY_DATA_DEFAULT = true export const emptyConfig: WorkflowsConfig = { isWorkflowRequired: false, + packageName: defaultPackageNameTemplates, successors: [], workflows: [getNoWorkflow({} as WorkflowsYaml, false)], } @@ -95,6 +120,10 @@ function parseWorkflow( isDefault: workflowSlug === data.default_workflow, isDisabled: false, name: workflow.name, + packageName: parsePackageNameTemplates( + data.catalog?.package_handle, + workflow.catalog?.package_handle, + ), schema: parseSchema(workflow.metadata_schema, data.schemas), slug: workflowSlug, } @@ -132,6 +161,7 @@ export function parse(workflowsYaml: string): WorkflowsConfig { const successors = data.successors || {} return { isWorkflowRequired: data.is_workflow_required !== false, + packageName: parsePackageNameTemplates(data.catalog?.package_handle), successors: Object.entries(successors).map(([url, successor]) => parseSuccessor(url, successor), ), From e5e5b2423454ea1ae7b30ee17cd7e6bf7b16085b Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 29 Sep 2021 16:45:17 +0300 Subject: [PATCH 12/55] add hook for getting package name template --- .../containers/Bucket/PackageCopyDialog.js | 31 +++++++++++++++++-- .../PackageDialog/PackageCreationForm.tsx | 21 +++++++++++-- .../Bucket/PackageDialog/PackageDialog.tsx | 30 ++++++++++++++++++ .../Bucket/PackageDirectoryDialog.tsx | 22 +++++++++---- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index c9c33d60f9..511bcdab6b 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -139,17 +139,42 @@ function DialogForm({ [nameWarning, nameExistence], ) + const [initialPackageName, onWorkflow] = PD.useInitialPackageName( + initialName || '', + selectedWorkflow || workflowsConfig, + ) + + const handleWorkflowChange = React.useCallback( + ({ modified, values }) => { + setWorkflow(values.workflow) + + if (modified.name) return + onWorkflow(values.workflow) + }, + [setWorkflow, onWorkflow], + ) + const [editorElement, setEditorElement] = React.useState() const onFormChange = React.useCallback( - async ({ values }) => { + async ({ modified, values }) => { if (document.body.contains(editorElement)) { setMetaHeight(editorElement.clientHeight) } + if (modified.workflow && values.workflow !== selectedWorkflow) { + handleWorkflowChange({ modified, values }) + } + handleNameChange(values.name) }, - [editorElement, handleNameChange, setMetaHeight], + [ + editorElement, + handleNameChange, + setMetaHeight, + selectedWorkflow, + handleWorkflowChange, + ], ) React.useEffect(() => { @@ -222,7 +247,7 @@ function DialogForm({ invalid: 'Invalid package name', }} helperText={nameWarning} - initialValue={initialName} + initialValue={initialPackageName} /> { + setWorkflow(values.workflow) + + if (modified.name) return + onWorkflow(values.workflow) + }, + [setWorkflow, onWorkflow], + ) + React.useEffect(() => { if (editorElement) resizeObserver.observe(editorElement) return () => { @@ -328,8 +343,8 @@ export function PackageCreationForm({ { - if (modified!.workflow) { - setWorkflow(values.workflow) + if (modified!.workflow && values.workflow !== selectedWorkflow) { + onWorkflowChange({ modified, values }) } }} /> @@ -354,7 +369,7 @@ export function PackageCreationForm({ + packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { + username: getUsernamePrefix(username), + }) + +export function useInitialPackageName( + initialName: string, + initialWorkflow: { packageName: packageHandleUtils.NameTemplates }, +): [string, (workflow: workflows.Workflow) => void] { + const username = redux.useSelector(authSelectors.username) + const [name, setName] = React.useState( + initialName || getDefaultPackageName(initialWorkflow, username), + ) + + const onWorkflow = React.useCallback( + (workflow: workflows.Workflow) => { + setName(getDefaultPackageName(workflow, username)) + }, + [setName, username], + ) + + return [name, onWorkflow] +} + const usePackageNameWarningStyles = M.makeStyles({ root: { marginRight: '4px', diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 87ef13b727..566c1e6215 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -3,11 +3,9 @@ import { basename } from 'path' import * as R from 'ramda' import * as React from 'react' import * as RF from 'react-final-form' -import * as redux from 'react-redux' import * as M from '@material-ui/core' import * as Intercom from 'components/Intercom' -import * as authSelectors from 'containers/Auth/selectors' import * as AWS from 'utils/AWS' import * as Data from 'utils/Data' import * as NamedRoutes from 'utils/NamedRoutes' @@ -203,8 +201,20 @@ function DialogForm({ } }, [editorElement, setMetaHeight]) - const username = redux.useSelector(authSelectors.username) - const usernamePrefix = React.useMemo(() => PD.getUsernamePrefix(username), [username]) + const [initialPackageName, onWorkflow] = PD.useInitialPackageName( + '', + selectedWorkflow || workflowsConfig, + ) + + const onWorkflowChange = React.useCallback( + ({ modified, values }) => { + setWorkflow(values.workflow) + + if (modified.name) return + onWorkflow(values.workflow) + }, + [setWorkflow, onWorkflow], + ) return ( { if (modified?.workflow && values.workflow !== selectedWorkflow) { - setWorkflow(values.workflow) + onWorkflowChange(values.workflow) } }} /> @@ -266,7 +276,7 @@ function DialogForm({ Date: Thu, 30 Sep 2021 13:46:57 +0300 Subject: [PATCH 13/55] minor changes --- catalog/app/utils/json-schema/json-schema.ts | 2 +- catalog/app/utils/packageHandle.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index d992029e1b..f4fbd26cd1 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -148,7 +148,7 @@ export function makeSchemaValidator( ...ajvOptions, } const ajv = new Ajv(options) - addFormats(ajv, ['uri']) + addFormats(ajv, ['date', 'uri']) try { if (!$id) return () => [new Error('$id is not provided')] diff --git a/catalog/app/utils/packageHandle.ts b/catalog/app/utils/packageHandle.ts index c0c4c8b1f4..a30baa6402 100644 --- a/catalog/app/utils/packageHandle.ts +++ b/catalog/app/utils/packageHandle.ts @@ -33,5 +33,6 @@ export function execTemplate( context: Context, options?: Options, ): string { + if (!templatesDict) return '' return execTemplateItem(templatesDict[context] || '', options) || '' } From 10017feb0ebdabe5def188a91609c9b0e8e15641 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 30 Sep 2021 15:59:41 +0300 Subject: [PATCH 14/55] use new approach for changing name on workflow change --- .../containers/Bucket/PackageCopyDialog.js | 33 +++------ .../PackageDialog/PackageCreationForm.tsx | 25 +++---- .../Bucket/PackageDialog/PackageDialog.tsx | 69 ++++++++++++------- .../Bucket/PackageDirectoryDialog.tsx | 21 ++---- 4 files changed, 66 insertions(+), 82 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index 511bcdab6b..f49d24e1ae 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -139,21 +139,6 @@ function DialogForm({ [nameWarning, nameExistence], ) - const [initialPackageName, onWorkflow] = PD.useInitialPackageName( - initialName || '', - selectedWorkflow || workflowsConfig, - ) - - const handleWorkflowChange = React.useCallback( - ({ modified, values }) => { - setWorkflow(values.workflow) - - if (modified.name) return - onWorkflow(values.workflow) - }, - [setWorkflow, onWorkflow], - ) - const [editorElement, setEditorElement] = React.useState() const onFormChange = React.useCallback( @@ -163,18 +148,17 @@ function DialogForm({ } if (modified.workflow && values.workflow !== selectedWorkflow) { - handleWorkflowChange({ modified, values }) + setWorkflow(values.workflow) } handleNameChange(values.name) }, - [ - editorElement, - handleNameChange, - setMetaHeight, - selectedWorkflow, - handleWorkflowChange, - ], + [editorElement, handleNameChange, selectedWorkflow, setMetaHeight, setWorkflow], + ) + + const getWorkflow = React.useCallback( + () => selectedWorkflow || workflowsConfig, + [selectedWorkflow, workflowsConfig], ) React.useEffect(() => { @@ -237,6 +221,7 @@ function DialogForm({ { - setWorkflow(values.workflow) - - if (modified.name) return - onWorkflow(values.workflow) - }, - [setWorkflow, onWorkflow], - ) - React.useEffect(() => { if (editorElement) resizeObserver.observe(editorElement) return () => { @@ -311,6 +296,11 @@ export function PackageCreationForm({ [delayHashing], ) + const getWorkflow = React.useCallback( + () => selectedWorkflow || workflowsConfig, + [selectedWorkflow, workflowsConfig], + ) + return ( { if (modified!.workflow && values.workflow !== selectedWorkflow) { - onWorkflowChange({ modified, values }) + setWorkflow(values.workflow) } }} /> @@ -369,7 +359,8 @@ export function PackageCreationForm({ input: RF.FieldInputProps + directory?: string meta: RF.FieldMetaState + getWorkflow: () => { packageName: packageHandleUtils.NameTemplates } validating: boolean } @@ -249,26 +254,53 @@ type PackageNameInputProps = PackageNameInputOwnProps & export function PackageNameInput({ errors, - input, + input: { value, onChange }, meta, + getWorkflow, + directory, validating, ...rest }: PackageNameInputProps) { - const readyForValidation = (input.value && meta.modified) || meta.submitFailed + const readyForValidation = (value && meta.modified) || meta.submitFailed const errorCode = readyForValidation && meta.error const error = errorCode ? errors[errorCode] || errorCode : '' + const [modified, setModified] = React.useState(meta.modified) + const handleChange = React.useCallback( + (event) => { + setModified(true) + onChange(event) + }, + [onChange, setModified], + ) const props = { disabled: meta.submitting || meta.submitSucceeded, error, fullWidth: true, label: 'Name', margin: 'normal' as const, + onChange: handleChange, placeholder: 'e.g. user/package', // NOTE: react-form doesn't change `FormState.validating` on async validation when field loses focus validating, - ...input, + value, ...rest, } + const username = redux.useSelector(authSelectors.username) + React.useEffect(() => { + if (modified) return + + const packageName = getDefaultPackageName(getWorkflow(), { + username, + directory, + }) + if (!packageName) return + + onChange({ + target: { + value: packageName, + }, + }) + }, [directory, getWorkflow, modified, onChange, username]) return } @@ -780,30 +812,15 @@ export function getUsernamePrefix(username?: string | null) { const getDefaultPackageName = ( workflow: { packageName: packageHandleUtils.NameTemplates }, - username: string, + { directory, username }: { directory?: string; username: string }, ) => - packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { - username: getUsernamePrefix(username), - }) - -export function useInitialPackageName( - initialName: string, - initialWorkflow: { packageName: packageHandleUtils.NameTemplates }, -): [string, (workflow: workflows.Workflow) => void] { - const username = redux.useSelector(authSelectors.username) - const [name, setName] = React.useState( - initialName || getDefaultPackageName(initialWorkflow, username), - ) - - const onWorkflow = React.useCallback( - (workflow: workflows.Workflow) => { - setName(getDefaultPackageName(workflow, username)) - }, - [setName, username], - ) - - return [name, onWorkflow] -} + directory + ? packageHandleUtils.execTemplate(workflow?.packageName, 'files', { + directory: basename(directory), + }) + : packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { + username: s3paths.ensureNoSlash(getUsernamePrefix(username)), + }) const usePackageNameWarningStyles = M.makeStyles({ root: { diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 566c1e6215..804cf7faa8 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -201,19 +201,9 @@ function DialogForm({ } }, [editorElement, setMetaHeight]) - const [initialPackageName, onWorkflow] = PD.useInitialPackageName( - '', - selectedWorkflow || workflowsConfig, - ) - - const onWorkflowChange = React.useCallback( - ({ modified, values }) => { - setWorkflow(values.workflow) - - if (modified.name) return - onWorkflow(values.workflow) - }, - [setWorkflow, onWorkflow], + const getWorkflow = React.useCallback( + () => selectedWorkflow || workflowsConfig, + [selectedWorkflow, workflowsConfig], ) return ( @@ -251,7 +241,7 @@ function DialogForm({ subscription={{ modified: true, values: true }} onChange={({ modified, values }) => { if (modified?.workflow && values.workflow !== selectedWorkflow) { - onWorkflowChange(values.workflow) + setWorkflow(values.workflow) } }} /> @@ -276,7 +266,8 @@ function DialogForm({ Date: Thu, 30 Sep 2021 16:11:55 +0300 Subject: [PATCH 15/55] add tests --- catalog/app/utils/packageHandle.spec.ts | 93 +++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 catalog/app/utils/packageHandle.spec.ts diff --git a/catalog/app/utils/packageHandle.spec.ts b/catalog/app/utils/packageHandle.spec.ts new file mode 100644 index 0000000000..3a56d24c77 --- /dev/null +++ b/catalog/app/utils/packageHandle.spec.ts @@ -0,0 +1,93 @@ +import * as packageHandle from './packageHandle' + +describe('utils/packageHandle', () => { + describe('convertItem', () => { + it('should return static string when no template', () => { + expect(packageHandle.execTemplateItem('fgsfds')).toBe('fgsfds') + expect(packageHandle.execTemplateItem('')).toBe('') + }) + + it('should treat broken template as a string and return this string', () => { + expect(packageHandle.execTemplateItem('start and <%= no end')).toBe( + 'start and <%= no end', + ) + }) + + it('should return converted when template has values', () => { + expect( + packageHandle.execTemplateItem( + 'what-<%=username %>-do/make-<%= directory %>-update', + { + directory: 'staging', + username: 'fiskus', + }, + ), + ).toBe('what-fiskus-do/make-staging-update') + expect( + packageHandle.execTemplateItem('<%= username %>/<%= directory %>', { + directory: 'staging', + username: 'fiskus', + }), + ).toBe('fiskus/staging') + }) + + it('should return null when no values ', () => { + expect( + packageHandle.execTemplateItem( + 'what-<%= username %>-do/make-<%= directory %>-update', + { + username: 'fiskus', + }, + ), + ).toBe(null) + expect( + packageHandle.execTemplateItem('<%= username %>/<%= directory %>', { + directory: 'staging', + }), + ).toBe(null) + }) + + it('should treat null/undefined as an empty string', () => { + expect( + packageHandle.execTemplateItem( + 'what-<%= username %>-do/make-<%= directory %>-update', + // @ts-expect-error + { directory: undefined, username: null }, + ), + ).toBe('what--do/make--update') + }) + }) + + describe('convert', () => { + it('should use contexted replacement', () => { + expect( + packageHandle.execTemplate( + { files: '<%= username %>/<%= directory %>', packages: 'abc/def' }, + 'packages', + ), + ).toBe('abc/def') + expect( + packageHandle.execTemplate( + { files: '<%= username %>/<%= directory %>', packages: '<%= a %>/<%= b %>' }, + 'files', + { + username: 'fiskus', + directory: 'staging', + }, + ), + ).toBe('fiskus/staging') + }) + + it('should return empty string if not enough values', () => { + expect( + packageHandle.execTemplate( + { files: '<%= username %>/<%= directory %>' }, + 'files', + { + username: 'fiskus', + }, + ), + ).toBe('') + }) + }) +}) From 1545cc74065cb5e5280fcc3f0886badc492a88f7 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 4 Oct 2021 09:13:06 +0300 Subject: [PATCH 16/55] add return type --- catalog/app/utils/json-schema/json-schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index f4fbd26cd1..fede58ac77 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -127,7 +127,7 @@ export function makeSchemaValidator( optSchema?: JsonSchema, optSchemas?: JsonSchema[], ajvOptions?: Options, -) { +): (obj?: any) => (Error | ErrorObject)[] { let mainSchema = R.clone(optSchema || EMPTY_SCHEMA) if (!mainSchema.$id) { // Make further code more universal by using one format: `id` → `$id` From 907bf406a7ef1ef082e6b9bf00888238ce4e6bd1 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 4 Oct 2021 10:26:31 +0300 Subject: [PATCH 17/55] add object_schema property --- .../app/containers/Bucket/requests/package.ts | 144 +++++++++++++----- catalog/app/utils/workflows.ts | 3 + shared/schemas/workflows.yml.json | 6 +- 3 files changed, 116 insertions(+), 37 deletions(-) diff --git a/catalog/app/containers/Bucket/requests/package.ts b/catalog/app/containers/Bucket/requests/package.ts index a61a3f421c..c6a6ff0582 100644 --- a/catalog/app/containers/Bucket/requests/package.ts +++ b/catalog/app/containers/Bucket/requests/package.ts @@ -1,3 +1,4 @@ +import type { ErrorObject } from 'ajv' import type { S3 } from 'aws-sdk' import * as R from 'ramda' import * as React from 'react' @@ -6,11 +7,67 @@ import { JsonValue } from 'components/JsonEditor/constants' import * as APIConnector from 'utils/APIConnector' import * as AWS from 'utils/AWS' import * as Config from 'utils/Config' -import { makeSchemaDefaultsSetter, JsonSchema } from 'utils/json-schema' +import { + makeSchemaDefaultsSetter, + makeSchemaValidator, + JsonSchema, +} from 'utils/json-schema' import mkSearch from 'utils/mkSearch' import pipeThru from 'utils/pipeThru' +import * as s3paths from 'utils/s3paths' import * as workflows from 'utils/workflows' +import * as errors from '../errors' +import * as requests from './requestsUntyped' + +export const objectSchema = async ({ s3, schemaUrl }: { s3: S3; schemaUrl: string }) => { + if (!schemaUrl) return null + + const { bucket, key, version } = s3paths.parseS3Url(schemaUrl) + + try { + const response = await requests.fetchFile({ s3, bucket, path: key, version }) + return JSON.parse(response.Body.toString('utf-8')) + } catch (e) { + if (e instanceof errors.FileNotFound || e instanceof errors.VersionNotFound) throw e + + // eslint-disable-next-line no-console + console.log('Unable to fetch') + // eslint-disable-next-line no-console + console.error(e) + } + + return null +} + +function formatErrorMessage(validationErrors: (ErrorObject | Error)[]): string { + const { instancePath, message = '' } = validationErrors[0] as ErrorObject + return instancePath ? `"${instancePath}" ${message}` : message +} + +async function validatePackageManifest( + s3: S3, + body: $TSFixMe, + schemaUrl?: string, +): Promise { + if (!schemaUrl) return undefined + + const schema = await objectSchema({ + s3, + schemaUrl, + }) + const normalizedBody = { + contents: body.entries || body.contents, + message: body.message, + meta: body.meta, + workflow: body.workflow, + } + const validationErrors = makeSchemaValidator(schema)(normalizedBody) + if (!validationErrors.length) return undefined + + return new Error(formatErrorMessage(validationErrors)) +} + interface AWSCredentials { accessKeyId: string secretAccessKey: string @@ -233,6 +290,17 @@ const mkCreatePackage = Body: payload, }) const res = await upload.promise() + + const error = await validatePackageManifest( + s3, + { + ...header, + contents, + }, + workflow.manifestSchema, + ) + if (error) throw error + return makeBackendRequest( req, ENDPOINT_CREATE, @@ -253,6 +321,7 @@ export function useCreatePackage() { } const copyPackage = async ( + s3: S3, req: ApiRequest, credentials: AWSCredentials, { message, meta, source, target, workflow }: CopyPackageParams, @@ -261,32 +330,33 @@ const copyPackage = async ( // refresh credentials and load if they are not loaded await credentials.getPromise() - return makeBackendRequest( - req, - ENDPOINT_COPY, - { - message, - meta: getMetaValue(meta, schema), - name: target.name, - parent: { - top_hash: source.revision, - registry: `s3://${source.bucket}`, - name: source.name, - }, - registry: `s3://${target.bucket}`, - workflow: getWorkflowApiParam(workflow.slug), + const body = { + message, + meta: getMetaValue(meta, schema), + name: target.name, + parent: { + top_hash: source.revision, + registry: `s3://${source.bucket}`, + name: source.name, }, - getCredentialsQuery(credentials), - ) + registry: `s3://${target.bucket}`, + workflow: getWorkflowApiParam(workflow.slug), + } + + const error = await validatePackageManifest(s3, body, workflow.manifestSchema) + if (error) throw error + + return makeBackendRequest(req, ENDPOINT_COPY, body, getCredentialsQuery(credentials)) } export function useCopyPackage() { const credentials = AWS.Credentials.use() const req: ApiRequest = APIConnector.use() + const s3 = AWS.S3.use() return React.useCallback( (params: CopyPackageParams, schema?: JsonSchema) => - copyPackage(req, credentials, params, schema), - [credentials, req], + copyPackage(s3, req, credentials, params, schema), + [credentials, req, s3], ) } @@ -320,6 +390,7 @@ export function useDeleteRevision() { } const wrapPackage = async ( + s3: S3, req: ApiRequest, credentials: AWSCredentials, { message, meta, source, target, workflow, entries }: WrapPackageParams, @@ -328,30 +399,31 @@ const wrapPackage = async ( // refresh credentials and load if they are not loaded await credentials.getPromise() - return makeBackendRequest( - req, - ENDPOINT_WRAP, - { - dst: { - registry: `s3://${target.bucket}`, - name: target.name, - }, - entries, - message, - meta: getMetaValue(meta, schema), - registry: `s3://${source}`, - workflow: getWorkflowApiParam(workflow.slug), + const body = { + dst: { + registry: `s3://${target.bucket}`, + name: target.name, }, - getCredentialsQuery(credentials), - ) + entries, + message, + meta: getMetaValue(meta, schema), + registry: `s3://${source}`, + workflow: getWorkflowApiParam(workflow.slug), + } + + const error = await validatePackageManifest(s3, body, workflow.manifestSchema) + if (error) throw error + + return makeBackendRequest(req, ENDPOINT_WRAP, body, getCredentialsQuery(credentials)) } export function useWrapPackage() { const credentials = AWS.Credentials.use() const req: ApiRequest = APIConnector.use() + const s3 = AWS.S3.use() return React.useCallback( (params: WrapPackageParams, schema?: JsonSchema) => - wrapPackage(req, credentials, params, schema), - [credentials, req], + wrapPackage(s3, req, credentials, params, schema), + [credentials, req, s3], ) } diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index 6f0eb38267..be0c179d2c 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -24,6 +24,7 @@ interface WorkflowYaml { description?: string is_message_required?: boolean metadata_schema?: string + object_schema?: string name: string catalog: { package_handle?: packageHandleUtils.NameTemplates @@ -50,6 +51,7 @@ export interface Workflow { description?: string isDefault: boolean isDisabled: boolean + manifestSchema?: string name?: string packageName: Required schema?: Schema @@ -119,6 +121,7 @@ function parseWorkflow( description: workflow.description, isDefault: workflowSlug === data.default_workflow, isDisabled: false, + manifestSchema: data.schemas?.[workflow.object_schema || '']?.url, name: workflow.name, packageName: parsePackageNameTemplates( data.catalog?.package_handle, diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 947c277c0f..81adaee4f0 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://schemas.quiltdata.com/workflows-config-1.0.0.json", + "$id": "https://schemas.quiltdata.com/workflows-config-2.0.0.json", "type": "object", "required": ["version", "workflows"], "additionalProperties": false, @@ -46,6 +46,10 @@ "type": "string", "description": "JSON Schema $id." }, + "object_schema": { + "type": "string", + "description": "JSON Schema $id." + }, "is_message_required": { "type": "boolean", "description": "If true, the user must provide a commit message.", From 7a83ab9f6358cacde23ec50de5332b05ee9e3805 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 4 Oct 2021 17:57:42 +0300 Subject: [PATCH 18/55] handle versioning too --- catalog/app/containers/Bucket/errors.tsx | 4 +++- catalog/app/utils/workflows.ts | 28 ++++++++++++++++++++--- catalog/package-lock.json | 14 ++++++++++++ catalog/package.json | 2 ++ shared/schemas/workflows-catalog.yml.json | 2 +- shared/schemas/workflows.yml.json | 2 +- 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/catalog/app/containers/Bucket/errors.tsx b/catalog/app/containers/Bucket/errors.tsx index 54b18c50e8..2da7269924 100644 --- a/catalog/app/containers/Bucket/errors.tsx +++ b/catalog/app/containers/Bucket/errors.tsx @@ -69,7 +69,9 @@ export class WorkflowsConfigInvalid extends BucketError { constructor(props: WorkflowsConfigInvalidProps) { super( props.errors - .map(({ instancePath, message }) => `${instancePath} ${message}`) + .map(({ instancePath, message }) => + instancePath ? `${instancePath} ${message}` : message, + ) .join(', '), props, ) diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index be0c179d2c..dc257d4c55 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -1,3 +1,5 @@ +import semver from 'semver' + import * as R from 'ramda' import { makeSchemaValidator } from 'utils/json-schema' @@ -139,11 +141,31 @@ const parseSuccessor = (url: string, successor: SuccessorYaml): Successor => ({ url, }) -const workflowsConfigValidator = makeSchemaValidator(workflowsCatalogConfigSchema, [ - workflowsConfigSchema, -]) +function validateConfigVersion(objectVersion: string): undefined | Error[] { + const baseSchemaVersion = '<=1.1.0' + const catalogSchemaVersion = '<=1.0.0' + + const [baseVersion, catalogVersion] = objectVersion.split(' ') + const errors = [] + if (!semver.satisfies(semver.coerce(baseVersion) || '0.0.0', baseSchemaVersion)) + errors.push(new Error(`Catalog supports ${baseSchemaVersion} version of base Schema`)) + if (!semver.satisfies(semver.coerce(catalogVersion) || '0.0.0', catalogSchemaVersion)) + errors.push( + new Error(`Catalog supports ${catalogSchemaVersion} version of catalog Schema`), + ) + return errors.length ? errors : undefined +} function validateConfig(data: unknown): asserts data is WorkflowsYaml { + const objectVersion = (data as WorkflowsYaml).version + const workflowsConfigValidator = makeSchemaValidator(workflowsCatalogConfigSchema, [ + workflowsConfigSchema, + ]) + + const versionErrors = validateConfigVersion(objectVersion) + if (versionErrors) + throw new bucketErrors.WorkflowsConfigInvalid({ errors: versionErrors }) + const errors = workflowsConfigValidator(data) if (errors.length) throw new bucketErrors.WorkflowsConfigInvalid({ errors }) } diff --git a/catalog/package-lock.json b/catalog/package-lock.json index 4490d016f0..9b0fe029b9 100644 --- a/catalog/package-lock.json +++ b/catalog/package-lock.json @@ -83,6 +83,7 @@ "remarkable": "^2.0.1", "reselect": "^4.0.0", "sanitize.css": "^12.0.1", + "semver": "^7.3.5", "tslib": "^2.3.1", "urql": "^2.0.4", "urql-custom-scalars-exchange": "^0.1.5", @@ -124,6 +125,7 @@ "@types/react-table": "^7.7.2", "@types/react-test-renderer": "^17.0.1", "@types/remarkable": "^2.0.3", + "@types/semver": "^7.3.8", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "bundlewatch": "^0.3.2", @@ -3456,6 +3458,12 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.1.tgz", "integrity": "sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA==" }, + "node_modules/@types/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-D/2EJvAlCEtYFEYmmlGwbGXuK886HzyCc3nZX/tkFTQdEU8jZDAgiv08P162yB17y4ZXZoq7yFAnW4GDBb9Now==", + "dev": true + }, "node_modules/@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", @@ -19983,6 +19991,12 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.1.tgz", "integrity": "sha512-EaCxbanVeyxDRTQBkdLb3Bvl/HK7PBK6UJjsSixB0iHKoWxE5uu2Q/DgtpOhPIojN0Zl1whvOd7PoHs2P0s5eA==" }, + "@types/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-D/2EJvAlCEtYFEYmmlGwbGXuK886HzyCc3nZX/tkFTQdEU8jZDAgiv08P162yB17y4ZXZoq7yFAnW4GDBb9Now==", + "dev": true + }, "@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", diff --git a/catalog/package.json b/catalog/package.json index d439850255..c51c000e50 100644 --- a/catalog/package.json +++ b/catalog/package.json @@ -127,6 +127,7 @@ "remarkable": "^2.0.1", "reselect": "^4.0.0", "sanitize.css": "^12.0.1", + "semver": "^7.3.5", "tslib": "^2.3.1", "urql": "^2.0.4", "urql-custom-scalars-exchange": "^0.1.5", @@ -168,6 +169,7 @@ "@types/react-table": "^7.7.2", "@types/react-test-renderer": "^17.0.1", "@types/remarkable": "^2.0.3", + "@types/semver": "^7.3.8", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "bundlewatch": "^0.3.2", diff --git a/shared/schemas/workflows-catalog.yml.json b/shared/schemas/workflows-catalog.yml.json index 3b2a29c50d..4be6e068e0 100644 --- a/shared/schemas/workflows-catalog.yml.json +++ b/shared/schemas/workflows-catalog.yml.json @@ -54,7 +54,7 @@ }, "allOf": [ { - "$ref": "https://schemas.quiltdata.com/workflows-config-1.0.0.json" + "$ref": "https://schemas.quiltdata.com/workflows-config-1.1.0.json" }, { "$ref": "#/definitions/CatalogSettings" diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 81adaee4f0..8f75ecb4b8 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://schemas.quiltdata.com/workflows-config-2.0.0.json", + "$id": "https://schemas.quiltdata.com/workflows-config-1.1.0.json", "type": "object", "required": ["version", "workflows"], "additionalProperties": false, From 3406757142b793ce0a906a89f3fbf848aa116aa4 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 5 Oct 2021 11:57:19 +0300 Subject: [PATCH 19/55] use handle_pattern template --- catalog/app/containers/Bucket/PackageCopyDialog.js | 2 +- .../Bucket/PackageDialog/PackageCreationForm.tsx | 2 +- .../Bucket/PackageDialog/PackageDialog.tsx | 8 ++++++-- .../app/containers/Bucket/PackageDirectoryDialog.tsx | 2 +- catalog/app/containers/Bucket/requests/package.ts | 6 +++--- catalog/app/utils/json-schema/json-schema.ts | 1 + catalog/app/utils/workflows.ts | 12 +++++++++--- shared/schemas/workflows.yml.json | 10 +++++++++- 8 files changed, 31 insertions(+), 12 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index f49d24e1ae..fe0f2b1619 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -81,7 +81,7 @@ function DialogForm({ validate: validateMetaInput, workflowsConfig, }) { - const nameValidator = PD.useNameValidator() + const nameValidator = PD.useNameValidator(selectedWorkflow) const nameExistence = PD.useNameExistence(successor.slug) const [nameWarning, setNameWarning] = React.useState('') const [metaHeight, setMetaHeight] = React.useState(0) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 3078ccf425..78cb375443 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -106,7 +106,7 @@ export function PackageCreationForm({ disableStateDisplay, ui = {}, }: PackageCreationFormProps & PD.SchemaFetcherRenderProps) { - const nameValidator = PD.useNameValidator() + const nameValidator = PD.useNameValidator(selectedWorkflow) const nameExistence = PD.useNameExistence(bucket) const [nameWarning, setNameWarning] = React.useState('') const [metaHeight, setMetaHeight] = React.useState(0) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index abe97835e9..aec473dec1 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -133,7 +133,7 @@ const validateName = (req: ApiRequest) => return undefined }, 200) -export function useNameValidator() { +export function useNameValidator(workflow?: workflows.Workflow) { const req: ApiRequest = APIConnector.use() const [counter, setCounter] = React.useState(0) const [processing, setProcessing] = React.useState(false) @@ -143,6 +143,10 @@ export function useNameValidator() { const validate = React.useCallback( async (name: string) => { + if (workflow?.packageNamePattern?.test(name) === false) { + return 'invalid' + } + setProcessing(true) try { const error = await validator(name) @@ -154,7 +158,7 @@ export function useNameValidator() { } }, // eslint-disable-next-line react-hooks/exhaustive-deps - [counter, validator], + [counter, validator, workflow], ) return React.useMemo(() => ({ validate, processing, inc }), [validate, processing, inc]) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 804cf7faa8..4a0a921182 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -95,7 +95,7 @@ function DialogForm({ validate: validateMetaInput, workflowsConfig, }: DialogFormProps & PD.SchemaFetcherRenderProps) { - const nameValidator = PD.useNameValidator() + const nameValidator = PD.useNameValidator(selectedWorkflow) const nameExistence = PD.useNameExistence(successor.slug) const [nameWarning, setNameWarning] = React.useState('') const [metaHeight, setMetaHeight] = React.useState(0) diff --git a/catalog/app/containers/Bucket/requests/package.ts b/catalog/app/containers/Bucket/requests/package.ts index c6a6ff0582..cc87fc5d3f 100644 --- a/catalog/app/containers/Bucket/requests/package.ts +++ b/catalog/app/containers/Bucket/requests/package.ts @@ -297,7 +297,7 @@ const mkCreatePackage = ...header, contents, }, - workflow.manifestSchema, + workflow.entriesSchema, ) if (error) throw error @@ -343,7 +343,7 @@ const copyPackage = async ( workflow: getWorkflowApiParam(workflow.slug), } - const error = await validatePackageManifest(s3, body, workflow.manifestSchema) + const error = await validatePackageManifest(s3, body, workflow.entriesSchema) if (error) throw error return makeBackendRequest(req, ENDPOINT_COPY, body, getCredentialsQuery(credentials)) @@ -411,7 +411,7 @@ const wrapPackage = async ( workflow: getWorkflowApiParam(workflow.slug), } - const error = await validatePackageManifest(s3, body, workflow.manifestSchema) + const error = await validatePackageManifest(s3, body, workflow.entriesSchema) if (error) throw error return makeBackendRequest(req, ENDPOINT_WRAP, body, getCredentialsQuery(credentials)) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index fede58ac77..9876fba8e5 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -149,6 +149,7 @@ export function makeSchemaValidator( } const ajv = new Ajv(options) addFormats(ajv, ['date', 'uri']) + ajv.addKeyword('dateformat') try { if (!$id) return () => [new Error('$id is not provided')] diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index dc257d4c55..09abcbe3ff 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -24,9 +24,10 @@ interface WorkflowsYaml { interface WorkflowYaml { description?: string + entries_schema?: string + handle_pattern?: string is_message_required?: boolean metadata_schema?: string - object_schema?: string name: string catalog: { package_handle?: packageHandleUtils.NameTemplates @@ -53,8 +54,9 @@ export interface Workflow { description?: string isDefault: boolean isDisabled: boolean - manifestSchema?: string + entriesSchema?: string name?: string + packageNamePattern: RegExp | null packageName: Required schema?: Schema slug: string | typeof notAvailable | typeof notSelected @@ -90,6 +92,7 @@ function getNoWorkflow(data: WorkflowsYaml, hasConfig: boolean): Workflow { isDefault: !data.default_workflow, isDisabled: data.is_workflow_required !== false, packageName: parsePackageNameTemplates(data.catalog?.package_handle), + packageNamePattern: null, slug: hasConfig ? notSelected : notAvailable, } } @@ -123,12 +126,15 @@ function parseWorkflow( description: workflow.description, isDefault: workflowSlug === data.default_workflow, isDisabled: false, - manifestSchema: data.schemas?.[workflow.object_schema || '']?.url, + entriesSchema: data.schemas?.[workflow.entries_schema || '']?.url, name: workflow.name, packageName: parsePackageNameTemplates( data.catalog?.package_handle, workflow.catalog?.package_handle, ), + packageNamePattern: workflow.handle_pattern + ? new RegExp(workflow.handle_pattern) + : null, schema: parseSchema(workflow.metadata_schema, data.schemas), slug: workflowSlug, } diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 8f75ecb4b8..9b3f8f84ea 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -42,11 +42,19 @@ "type": "string", "minLength": 1 }, + "handle_pattern": { + "type": "string", + "description": "Regular expression to validate package handle", + "examples": [ + "instruments/(production/staging)", + "(employee1|employee2)/instruments" + ] + }, "metadata_schema": { "type": "string", "description": "JSON Schema $id." }, - "object_schema": { + "entries_schema": { "type": "string", "description": "JSON Schema $id." }, From 191f1f229f17d745a8436b5db6b0f760f9821b4e Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 5 Oct 2021 12:11:54 +0300 Subject: [PATCH 20/55] show pattern error --- catalog/app/containers/Bucket/PackageCopyDialog.js | 1 + .../app/containers/Bucket/PackageDialog/PackageCreationForm.tsx | 1 + catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx | 2 +- catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index fe0f2b1619..68a7d9ab06 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -230,6 +230,7 @@ function DialogForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', + pattern: `Name should match with ${selectedWorkflow?.packageNamePattern}`, }} helperText={nameWarning} initialValue={initialName} diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 78cb375443..53677c87e2 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -370,6 +370,7 @@ export function PackageCreationForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', + pattern: `Name should match with ${selectedWorkflow?.packageNamePattern}`, }} helperText={nameWarning} validating={nameValidator.processing} diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index aec473dec1..49cb1c811f 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -144,7 +144,7 @@ export function useNameValidator(workflow?: workflows.Workflow) { const validate = React.useCallback( async (name: string) => { if (workflow?.packageNamePattern?.test(name) === false) { - return 'invalid' + return 'pattern' } setProcessing(true) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 4a0a921182..921b8e7ff7 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -277,6 +277,7 @@ function DialogForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', + pattern: `Name should match with "${selectedWorkflow?.packageNamePattern}" regexp`, }} helperText={nameWarning} /> From 7c7672503f8a488312f5a463cc600ea69815d6de Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Tue, 5 Oct 2021 15:25:06 +0300 Subject: [PATCH 21/55] add entries validator --- .../PackageDialog/PackageCreationForm.tsx | 32 ++++++++++++++++++- .../Bucket/PackageDialog/PackageDialog.tsx | 17 ++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 53677c87e2..0999e54ef9 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -1,3 +1,4 @@ +import type { ErrorObject } from 'ajv' import * as FF from 'final-form' import * as FP from 'fp-ts' import * as R from 'ramda' @@ -18,6 +19,7 @@ import DialogLoading from './DialogLoading' import DialogSuccess, { DialogSuccessRenderMessageProps } from './DialogSuccess' import * as FI from './FilesInput' import * as Layout from './Layout' +import MetaInputErrorHelper from './MetaInputErrorHelper' import * as PD from './PackageDialog' import { isS3File, S3File } from './S3FilePicker' import { FormSkeleton, MetaInputSkeleton } from './Skeleton' @@ -56,6 +58,9 @@ const useStyles = M.makeStyles((t) => ({ files: { height: '100%', }, + filesError: { + marginTop: t.spacing(), + }, form: { height: '100%', }, @@ -114,6 +119,10 @@ export function PackageCreationForm({ const dialogContentClasses = PD.useContentStyles({ metaHeight }) const validateWorkflow = PD.useWorkflowValidator(workflowsConfig) + const [entriesError, setEntriesError] = React.useState<(Error | ErrorObject)[] | null>( + null, + ) + const [selectedBucket, selectBucket] = React.useState(sourceBuckets.getDefault) const existingEntries = initial?.manifest?.entries ?? EMPTY_MANIFEST_ENTRIES @@ -137,6 +146,7 @@ export function PackageCreationForm({ ) const createPackage = requests.useCreatePackage() + const validateEntries = PD.useEntriesValidator(selectedWorkflow) const onSubmit = async ({ name, @@ -167,6 +177,20 @@ export function PackageCreationForm({ return !e || e.hash !== file.hash.value }) + const entries = toUpload.map(({ file, path }) => ({ + logical_key: path, + size: file.size, + })) + const error = await validateEntries(entries) + // console.log({ toUpload, entriesError, entries }) + if (error) { + setEntriesError(error) + return { + files: 'schema', + } + } + return + let uploadedEntries try { uploadedEntries = await uploads.upload({ @@ -230,7 +254,7 @@ export function PackageCreationForm({ // eslint-disable-next-line no-console console.log('error creating manifest', e) // TODO: handle specific cases? - const errorMessage = e instanceof Error ? e.message : null + const errorMessage = e instanceof Error ? (e as Error).message : null return { [FF.FORM_ERROR]: errorMessage || PD.ERROR_MESSAGES.MANIFEST } } } @@ -415,6 +439,7 @@ export function PackageCreationForm({ validateFields={['files']} errors={{ nonEmpty: 'Add files to create a package', + schema: 'Files should match schema', [FI.HASHING]: 'Please wait while we hash the files', [FI.HASHING_ERROR]: 'Error hashing files, probably some of them are too large. Please try again or contact support.', @@ -431,6 +456,11 @@ export function PackageCreationForm({ disableStateDisplay={disableStateDisplay} ui={{ reset: ui.resetFiles }} /> + + diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index 49cb1c811f..8b6c85938f 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -870,3 +870,20 @@ export function DialogWrapper({ ) return } + +export function useEntriesValidator(workflow?: workflows.Workflow) { + const s3 = AWS.S3.use() + + return React.useCallback( + async (entries: $TSFixMe) => { + const schemaUrl = workflow?.entriesSchema + if (!schemaUrl) return undefined + const entriesSchema = await requests.objectSchema({ s3, schemaUrl }) + // TODO: Show error if there is network error + if (!entriesSchema) return undefined + + return makeSchemaValidator(entriesSchema)(entries) + }, + [workflow, s3], + ) +} From 956ad11febfb54a32dce3be085acdfb1f3f176df Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 6 Oct 2021 14:18:10 +0300 Subject: [PATCH 22/55] adjust UI --- .../app/containers/Bucket/PackageDialog/PackageCreationForm.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 0999e54ef9..bbb18d70c1 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -60,6 +60,8 @@ const useStyles = M.makeStyles((t) => ({ }, filesError: { marginTop: t.spacing(), + maxHeight: t.spacing(9), + overflowY: 'auto', }, form: { height: '100%', From 728f98cf611d70bafc86687770fdf5c79d1d4cb0 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Wed, 6 Oct 2021 18:21:27 +0300 Subject: [PATCH 23/55] show files error --- .../PackageDialog/PackageCreationForm.tsx | 10 +++-- .../Bucket/PackageDirectoryDialog.tsx | 41 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index bbb18d70c1..19f97ef96a 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -1,4 +1,5 @@ import type { ErrorObject } from 'ajv' +import cx from 'classnames' import * as FF from 'final-form' import * as FP from 'fp-ts' import * as R from 'ramda' @@ -58,6 +59,9 @@ const useStyles = M.makeStyles((t) => ({ files: { height: '100%', }, + filesWithError: { + height: `calc(90% - ${t.spacing()}px)`, + }, filesError: { marginTop: t.spacing(), maxHeight: t.spacing(9), @@ -184,14 +188,12 @@ export function PackageCreationForm({ size: file.size, })) const error = await validateEntries(entries) - // console.log({ toUpload, entriesError, entries }) if (error) { setEntriesError(error) return { files: 'schema', } } - return let uploadedEntries try { @@ -433,7 +435,9 @@ export function PackageCreationForm({ { @@ -52,6 +55,14 @@ const useStyles = M.makeStyles((t) => ({ files: { height: '100%', }, + filesWithError: { + height: `calc(90% - ${t.spacing()}px)`, + }, + filesError: { + marginTop: t.spacing(), + maxHeight: t.spacing(9), + overflowY: 'auto', + }, form: { height: '100%', }, @@ -100,8 +111,13 @@ function DialogForm({ const [nameWarning, setNameWarning] = React.useState('') const [metaHeight, setMetaHeight] = React.useState(0) const validateWorkflow = PD.useWorkflowValidator(workflowsConfig) + const validateEntries = PD.useEntriesValidator(selectedWorkflow) const classes = useStyles() + const [entriesError, setEntriesError] = React.useState<(Error | ErrorObject)[] | null>( + null, + ) + const createPackage = requests.useWrapPackage() const dialogContentClasses = PD.useContentStyles({ metaHeight }) @@ -120,6 +136,20 @@ function DialogForm({ // eslint-disable-next-line consistent-return }) => { try { + const entries = filesValue + .filter((f) => f.selected) + .map((f) => ({ + logical_key: f.name, + size: f.size, + })) + const error = await validateEntries(entries) + if (error) { + setEntriesError(error) + return { + files: 'schema', + } + } + const res = await createPackage( { ...values, @@ -141,7 +171,7 @@ function DialogForm({ return { [FF.FORM_ERROR]: errorMessage || PD.ERROR_MESSAGES.MANIFEST } } }, - [bucket, successor, createPackage, setSuccess, schema, path], + [bucket, successor, createPackage, setSuccess, schema, path, validateEntries], ) const initialFiles: PD.FilesSelectorState = React.useMemo( @@ -316,7 +346,9 @@ function DialogForm({ + + From 919b335502dae94869eb29d9ef8ea23078c346c5 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 08:44:48 +0300 Subject: [PATCH 24/55] clear error message on files add --- .../containers/Bucket/PackageDialog/PackageCreationForm.tsx | 1 + catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 19f97ef96a..196590ba78 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -297,6 +297,7 @@ export function PackageCreationForm({ const onFormChange = React.useCallback( async ({ dirtyFields, values }) => { if (dirtyFields?.name) handleNameChange(values.name) + if (dirtyFields?.files) setEntriesError(null) }, [handleNameChange], ) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 67bf99f8ad..fc7ab3547c 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -215,11 +215,13 @@ function DialogForm({ const [editorElement, setEditorElement] = React.useState(null) const onFormChange = React.useCallback( - async ({ values }) => { + async ({ dirtyFields, values }) => { if (document.body.contains(editorElement)) { setMetaHeight(editorElement!.clientHeight) } + if (dirtyFields?.files) setEntriesError(null) + handleNameChange(values.name) }, [editorElement, handleNameChange, setMetaHeight], From 1d9372f6af4fd920f6fcbbb51ea23e0a3fbef3c9 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 10:02:54 +0300 Subject: [PATCH 25/55] Update shared/schemas/workflows.yml.json Co-authored-by: Sergey Fedoseev --- shared/schemas/workflows.yml.json | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows.yml.json index 9b3f8f84ea..c56f5badb8 100644 --- a/shared/schemas/workflows.yml.json +++ b/shared/schemas/workflows.yml.json @@ -44,6 +44,7 @@ }, "handle_pattern": { "type": "string", + "format": "regex", "description": "Regular expression to validate package handle", "examples": [ "instruments/(production/staging)", From 047393e4dc76d7479b13749a33cfd88ed8864ef3 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 10:03:27 +0300 Subject: [PATCH 26/55] use object version --- catalog/app/utils/workflows.ts | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index 09abcbe3ff..8caac6bdfd 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -10,6 +10,11 @@ import workflowsConfigSchema from 'schemas/workflows.yml.json' import workflowsCatalogConfigSchema from 'schemas/workflows-catalog.yml.json' import * as bucketErrors from 'containers/Bucket/errors' +interface WorkflowsVersion { + base: string + catalog?: string +} + interface WorkflowsYaml { default_workflow?: string is_workflow_required?: boolean @@ -18,7 +23,7 @@ interface WorkflowsYaml { } schemas?: Record successors?: Record - version: '1' + version: '1' | WorkflowsVersion workflows: Record } @@ -147,28 +152,35 @@ const parseSuccessor = (url: string, successor: SuccessorYaml): Successor => ({ url, }) -function validateConfigVersion(objectVersion: string): undefined | Error[] { +function validateConfigVersion(objectVersion: WorkflowsVersion): undefined | Error[] { const baseSchemaVersion = '<=1.1.0' const catalogSchemaVersion = '<=1.0.0' - const [baseVersion, catalogVersion] = objectVersion.split(' ') + const { base: baseVersion, catalog: catalogVersion } = objectVersion const errors = [] - if (!semver.satisfies(semver.coerce(baseVersion) || '0.0.0', baseSchemaVersion)) + if (!semver.satisfies(semver.coerce(baseVersion) || '0.0.0', baseSchemaVersion)) { errors.push(new Error(`Catalog supports ${baseSchemaVersion} version of base Schema`)) - if (!semver.satisfies(semver.coerce(catalogVersion) || '0.0.0', catalogSchemaVersion)) + } + if ( + catalogVersion && + !semver.satisfies(semver.coerce(catalogVersion) || '0.0.0', catalogSchemaVersion) + ) { errors.push( new Error(`Catalog supports ${catalogSchemaVersion} version of catalog Schema`), ) + } return errors.length ? errors : undefined } function validateConfig(data: unknown): asserts data is WorkflowsYaml { const objectVersion = (data as WorkflowsYaml).version - const workflowsConfigValidator = makeSchemaValidator(workflowsCatalogConfigSchema, [ - workflowsConfigSchema, - ]) + const workflowsConfigValidator = (objectVersion as WorkflowsVersion).catalog + ? makeSchemaValidator(workflowsCatalogConfigSchema, [workflowsConfigSchema]) + : makeSchemaValidator(workflowsConfigSchema) - const versionErrors = validateConfigVersion(objectVersion) + const versionErrors = validateConfigVersion( + typeof objectVersion === 'string' ? { base: objectVersion } : objectVersion, + ) if (versionErrors) throw new bucketErrors.WorkflowsConfigInvalid({ errors: versionErrors }) From 5607921ff1e292e72df36edad5f31bf7df126b15 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 10:04:19 +0300 Subject: [PATCH 27/55] use regex format too --- catalog/app/utils/json-schema/json-schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index 9876fba8e5..71acd3f7c3 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -148,7 +148,7 @@ export function makeSchemaValidator( ...ajvOptions, } const ajv = new Ajv(options) - addFormats(ajv, ['date', 'uri']) + addFormats(ajv, ['date', 'regex', 'uri']) ajv.addKeyword('dateformat') try { From 85ed54adc631cbc00800ef3820aad3d21a188d28 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 12:25:46 +0300 Subject: [PATCH 28/55] rename schemas --- catalog/app/utils/workflows.ts | 4 ++-- .../{workflows.yml.json => workflows-config-1.1.0.json} | 0 ...s-catalog.yml.json => workflows-config_catalog-1.0.0.json} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename shared/schemas/{workflows.yml.json => workflows-config-1.1.0.json} (100%) rename shared/schemas/{workflows-catalog.yml.json => workflows-config_catalog-1.0.0.json} (100%) diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index 8caac6bdfd..7581bdbcf6 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -6,8 +6,8 @@ import { makeSchemaValidator } from 'utils/json-schema' import type * as packageHandleUtils from 'utils/packageHandle' import * as s3paths from 'utils/s3paths' import yaml from 'utils/yaml' -import workflowsConfigSchema from 'schemas/workflows.yml.json' -import workflowsCatalogConfigSchema from 'schemas/workflows-catalog.yml.json' +import workflowsConfigSchema from 'schemas/workflows-config-1.1.0.json' +import workflowsCatalogConfigSchema from 'schemas/workflows-config_catalog-1.0.0.json' import * as bucketErrors from 'containers/Bucket/errors' interface WorkflowsVersion { diff --git a/shared/schemas/workflows.yml.json b/shared/schemas/workflows-config-1.1.0.json similarity index 100% rename from shared/schemas/workflows.yml.json rename to shared/schemas/workflows-config-1.1.0.json diff --git a/shared/schemas/workflows-catalog.yml.json b/shared/schemas/workflows-config_catalog-1.0.0.json similarity index 100% rename from shared/schemas/workflows-catalog.yml.json rename to shared/schemas/workflows-config_catalog-1.0.0.json From 0b0805e9c47009dcb457afa8279b0c0e4334ba41 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 14:51:20 +0300 Subject: [PATCH 29/55] add previous version --- shared/schemas/workflows-config-1.json | 118 +++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 shared/schemas/workflows-config-1.json diff --git a/shared/schemas/workflows-config-1.json b/shared/schemas/workflows-config-1.json new file mode 100644 index 0000000000..7c14ef8975 --- /dev/null +++ b/shared/schemas/workflows-config-1.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://quiltdata.com/workflows/config/1", + "type": "object", + "required": [ + "version", + "workflows" + ], + "additionalProperties": false, + "properties": { + "version": { + "const": "1" + }, + "is_workflow_required": { + "type": "boolean", + "description": "If true, users must succeed a workflow in order to push. If false, users may skip workflows altogether.", + "default": true + }, + "default_workflow": { + "type": "string", + "description": "The workflow to use if the user doesn't specify a workflow." + }, + "workflows": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[A-Za-z_-][A-Za-z0-9_-]*$", + "maxLength": 64 + }, + "additionalProperties": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The workflow name displayed by Quilt in the Python API or web UI.", + "minLength": 1 + }, + "description": { + "type": "string", + "description": "The workflow description displayed by Quilt in the Python API or web UI.", + "minLength": 1 + }, + "metadata_schema": { + "type": "string", + "description": "JSON Schema $id." + }, + "is_message_required": { + "type": "boolean", + "description": "If true, the user must provide a commit message.", + "default": false + } + } + } + }, + "schemas": { + "type": "object", + "description": "JSON Schemas for validating user-supplied metadata.", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "url": { + "type": "string", + "description": "URL from where the schema will be obtained.", + "format": "uri" + } + } + } + }, + "successors": { + "type": "object", + "description": "Buckets usable as destination with \"Push to Bucket\" in web UI.", + "examples": [ + { + "s3://bucket1": { + "title": "Bucket 1 Title", + "copy_data": true + }, + "s3://bucket2": { + "title": "Bucket 2 Title", + "copy_data": false + }, + "s3://bucket3": { + "title": "Bucket 3 Title (`copy_data` defaults to `true`)" + } + } + ], + "minProperties": 1, + "propertyNames": { + "format": "uri" + }, + "additionalProperties": { + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "type": "string", + "minLength": 1 + }, + "copy_data": { + "type": "boolean", + "default": true, + "description": "If true, all package entries will be copied to the destination bucket. If false, all entries will remain in their current locations." + } + } + } + } + } +} + From fab09e5b01cf2f95c0e0f32c9452770f803e3e42 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 17:10:26 +0300 Subject: [PATCH 30/55] fix uploading valid package --- .../containers/Bucket/PackageDialog/PackageCreationForm.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 196590ba78..7d2ed9770e 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -188,7 +188,7 @@ export function PackageCreationForm({ size: file.size, })) const error = await validateEntries(entries) - if (error) { + if (error && error.length) { setEntriesError(error) return { files: 'schema', @@ -437,7 +437,7 @@ export function PackageCreationForm({ Date: Thu, 7 Oct 2021 17:18:10 +0300 Subject: [PATCH 31/55] Remove extra empty line --- shared/schemas/workflows-config-1.json | 1 - 1 file changed, 1 deletion(-) diff --git a/shared/schemas/workflows-config-1.json b/shared/schemas/workflows-config-1.json index 7c14ef8975..ba95c7be2d 100644 --- a/shared/schemas/workflows-config-1.json +++ b/shared/schemas/workflows-config-1.json @@ -115,4 +115,3 @@ } } } - From a5c6e72f37923fa583bf694f57b3e685ab08109b Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 7 Oct 2021 17:29:44 +0300 Subject: [PATCH 32/55] remove outdated code --- .../app/containers/Bucket/requests/package.ts | 63 ++----------------- 1 file changed, 5 insertions(+), 58 deletions(-) diff --git a/catalog/app/containers/Bucket/requests/package.ts b/catalog/app/containers/Bucket/requests/package.ts index cc87fc5d3f..6397b74eed 100644 --- a/catalog/app/containers/Bucket/requests/package.ts +++ b/catalog/app/containers/Bucket/requests/package.ts @@ -1,4 +1,3 @@ -import type { ErrorObject } from 'ajv' import type { S3 } from 'aws-sdk' import * as R from 'ramda' import * as React from 'react' @@ -7,11 +6,7 @@ import { JsonValue } from 'components/JsonEditor/constants' import * as APIConnector from 'utils/APIConnector' import * as AWS from 'utils/AWS' import * as Config from 'utils/Config' -import { - makeSchemaDefaultsSetter, - makeSchemaValidator, - JsonSchema, -} from 'utils/json-schema' +import { makeSchemaDefaultsSetter, JsonSchema } from 'utils/json-schema' import mkSearch from 'utils/mkSearch' import pipeThru from 'utils/pipeThru' import * as s3paths from 'utils/s3paths' @@ -40,34 +35,6 @@ export const objectSchema = async ({ s3, schemaUrl }: { s3: S3; schemaUrl: strin return null } -function formatErrorMessage(validationErrors: (ErrorObject | Error)[]): string { - const { instancePath, message = '' } = validationErrors[0] as ErrorObject - return instancePath ? `"${instancePath}" ${message}` : message -} - -async function validatePackageManifest( - s3: S3, - body: $TSFixMe, - schemaUrl?: string, -): Promise { - if (!schemaUrl) return undefined - - const schema = await objectSchema({ - s3, - schemaUrl, - }) - const normalizedBody = { - contents: body.entries || body.contents, - message: body.message, - meta: body.meta, - workflow: body.workflow, - } - const validationErrors = makeSchemaValidator(schema)(normalizedBody) - if (!validationErrors.length) return undefined - - return new Error(formatErrorMessage(validationErrors)) -} - interface AWSCredentials { accessKeyId: string secretAccessKey: string @@ -291,16 +258,6 @@ const mkCreatePackage = }) const res = await upload.promise() - const error = await validatePackageManifest( - s3, - { - ...header, - contents, - }, - workflow.entriesSchema, - ) - if (error) throw error - return makeBackendRequest( req, ENDPOINT_CREATE, @@ -321,7 +278,6 @@ export function useCreatePackage() { } const copyPackage = async ( - s3: S3, req: ApiRequest, credentials: AWSCredentials, { message, meta, source, target, workflow }: CopyPackageParams, @@ -343,20 +299,16 @@ const copyPackage = async ( workflow: getWorkflowApiParam(workflow.slug), } - const error = await validatePackageManifest(s3, body, workflow.entriesSchema) - if (error) throw error - return makeBackendRequest(req, ENDPOINT_COPY, body, getCredentialsQuery(credentials)) } export function useCopyPackage() { const credentials = AWS.Credentials.use() const req: ApiRequest = APIConnector.use() - const s3 = AWS.S3.use() return React.useCallback( (params: CopyPackageParams, schema?: JsonSchema) => - copyPackage(s3, req, credentials, params, schema), - [credentials, req, s3], + copyPackage(req, credentials, params, schema), + [credentials, req], ) } @@ -390,7 +342,6 @@ export function useDeleteRevision() { } const wrapPackage = async ( - s3: S3, req: ApiRequest, credentials: AWSCredentials, { message, meta, source, target, workflow, entries }: WrapPackageParams, @@ -411,19 +362,15 @@ const wrapPackage = async ( workflow: getWorkflowApiParam(workflow.slug), } - const error = await validatePackageManifest(s3, body, workflow.entriesSchema) - if (error) throw error - return makeBackendRequest(req, ENDPOINT_WRAP, body, getCredentialsQuery(credentials)) } export function useWrapPackage() { const credentials = AWS.Credentials.use() const req: ApiRequest = APIConnector.use() - const s3 = AWS.S3.use() return React.useCallback( (params: WrapPackageParams, schema?: JsonSchema) => - wrapPackage(s3, req, credentials, params, schema), - [credentials, req, s3], + wrapPackage(req, credentials, params, schema), + [credentials, req], ) } From 985899966c2b11084439af24ff3a1b3001a2bfde Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 09:38:19 +0300 Subject: [PATCH 33/55] add description --- shared/schemas/workflows-config_catalog-1.0.0.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shared/schemas/workflows-config_catalog-1.0.0.json b/shared/schemas/workflows-config_catalog-1.0.0.json index 4be6e068e0..d5c4a7acc9 100644 --- a/shared/schemas/workflows-config_catalog-1.0.0.json +++ b/shared/schemas/workflows-config_catalog-1.0.0.json @@ -13,10 +13,12 @@ "additionalProperties": false, "properties": { "files": { - "$ref": "#/definitions/PackageHandleTemplate" + "$ref": "#/definitions/PackageHandleTemplate", + "description": "Default package name used for creating package from directory" }, "packages": { - "$ref": "#/definitions/PackageHandleTemplate" + "$ref": "#/definitions/PackageHandleTemplate", + "description": "Default package name used for creating package from scratch or from another package" } } }, From 42825cec1da8f63c38725f73647fe0506afcbc0c Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 09:42:51 +0300 Subject: [PATCH 34/55] Apply suggestions from code review Co-authored-by: Alexei Mochalov --- catalog/app/containers/Bucket/PackageCopyDialog.js | 2 +- .../app/containers/Bucket/PackageDialog/PackageCreationForm.tsx | 2 +- catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx | 2 +- catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 2 +- shared/schemas/workflows-config_catalog-1.0.0.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index 68a7d9ab06..61e0508ade 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -230,7 +230,7 @@ function DialogForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', - pattern: `Name should match with ${selectedWorkflow?.packageNamePattern}`, + pattern: `Name should match ${selectedWorkflow?.packageNamePattern}`, }} helperText={nameWarning} initialValue={initialName} diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 7d2ed9770e..6e54d2897e 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -399,7 +399,7 @@ export function PackageCreationForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', - pattern: `Name should match with ${selectedWorkflow?.packageNamePattern}`, + pattern: `Name should match ${selectedWorkflow?.packageNamePattern}`, }} helperText={nameWarning} validating={nameValidator.processing} diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index 8b6c85938f..56db995e06 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -158,7 +158,7 @@ export function useNameValidator(workflow?: workflows.Workflow) { } }, // eslint-disable-next-line react-hooks/exhaustive-deps - [counter, validator, workflow], + [counter, validator, workflow?.packageNamePattern], ) return React.useMemo(() => ({ validate, processing, inc }), [validate, processing, inc]) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index fc7ab3547c..68e614be4e 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -309,7 +309,7 @@ function DialogForm({ errors={{ required: 'Enter a package name', invalid: 'Invalid package name', - pattern: `Name should match with "${selectedWorkflow?.packageNamePattern}" regexp`, + pattern: `Name should match "${selectedWorkflow?.packageNamePattern}" regexp`, }} helperText={nameWarning} /> diff --git a/shared/schemas/workflows-config_catalog-1.0.0.json b/shared/schemas/workflows-config_catalog-1.0.0.json index 4be6e068e0..f613a37d66 100644 --- a/shared/schemas/workflows-config_catalog-1.0.0.json +++ b/shared/schemas/workflows-config_catalog-1.0.0.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://schemas.quiltdata.com/workflows-config-catalog-1.0.0.json", + "$id": "https://schemas.quiltdata.com/workflows-config_catalog-1.0.0", "definitions": { "PackageHandleTemplate": { "description": "Template for pre-filling package handle, use <%= username %> or <%= directory %> for username and parent directory substitutions", From 846d4fe9377852844d118db6db5c69ea185dfc80 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 09:43:43 +0300 Subject: [PATCH 35/55] omit file extension --- shared/schemas/workflows-config-1.1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/schemas/workflows-config-1.1.0.json b/shared/schemas/workflows-config-1.1.0.json index c56f5badb8..0130c2348a 100644 --- a/shared/schemas/workflows-config-1.1.0.json +++ b/shared/schemas/workflows-config-1.1.0.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://schemas.quiltdata.com/workflows-config-1.1.0.json", + "$id": "https://schemas.quiltdata.com/workflows-config-1.1.0", "type": "object", "required": ["version", "workflows"], "additionalProperties": false, From e5856c9345be454372598da3abd8a33ceeed1938 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 09:46:05 +0300 Subject: [PATCH 36/55] update example --- shared/schemas/workflows-config-1.1.0.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/schemas/workflows-config-1.1.0.json b/shared/schemas/workflows-config-1.1.0.json index 0130c2348a..3cade0250f 100644 --- a/shared/schemas/workflows-config-1.1.0.json +++ b/shared/schemas/workflows-config-1.1.0.json @@ -47,8 +47,8 @@ "format": "regex", "description": "Regular expression to validate package handle", "examples": [ - "instruments/(production/staging)", - "(employee1|employee2)/instruments" + "^instruments/(production/staging)$", + "^(employee1|employee2)/instruments$" ] }, "metadata_schema": { From 804751b3544aa365e071c835e486dabe7fb431c0 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 09:52:19 +0300 Subject: [PATCH 37/55] add console.error --- catalog/app/utils/packageHandle.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalog/app/utils/packageHandle.ts b/catalog/app/utils/packageHandle.ts index a30baa6402..151adab43c 100644 --- a/catalog/app/utils/packageHandle.ts +++ b/catalog/app/utils/packageHandle.ts @@ -24,6 +24,10 @@ export function execTemplateItem(template: string, options?: Options): string | try { return lodashTemplate(template)(options) } catch (error) { + // eslint-disable-next-line no-console + console.log('Template for default package name is invalid') + // eslint-disable-next-line no-console + console.error(error) return null } } From 4df465c9e94ce897f9a2735d7d18a588068dbe50 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 11:24:38 +0300 Subject: [PATCH 38/55] hide console.logs --- catalog/app/utils/packageHandle.spec.ts | 31 ++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/catalog/app/utils/packageHandle.spec.ts b/catalog/app/utils/packageHandle.spec.ts index 3a56d24c77..9f6e864c4d 100644 --- a/catalog/app/utils/packageHandle.spec.ts +++ b/catalog/app/utils/packageHandle.spec.ts @@ -1,5 +1,7 @@ import * as packageHandle from './packageHandle' +/* eslint-disable no-console */ + describe('utils/packageHandle', () => { describe('convertItem', () => { it('should return static string when no template', () => { @@ -31,7 +33,9 @@ describe('utils/packageHandle', () => { ).toBe('fiskus/staging') }) - it('should return null when no values ', () => { + it('should return null when no value for directory', () => { + console.log = jest.fn() + console.error = jest.fn() expect( packageHandle.execTemplateItem( 'what-<%= username %>-do/make-<%= directory %>-update', @@ -40,11 +44,28 @@ describe('utils/packageHandle', () => { }, ), ).toBe(null) + expect(console.log).toHaveBeenCalledWith( + 'Template for default package name is invalid', + ) + expect((console.error as jest.Mock).mock.calls[0][0]).toMatchObject({ + message: 'directory is not defined', + }) + }) + + it('should return null when no value for username', () => { + console.log = jest.fn() + console.error = jest.fn() expect( packageHandle.execTemplateItem('<%= username %>/<%= directory %>', { directory: 'staging', }), ).toBe(null) + expect(console.log).toHaveBeenCalledWith( + 'Template for default package name is invalid', + ) + expect((console.error as jest.Mock).mock.calls[0][0]).toMatchObject({ + message: 'username is not defined', + }) }) it('should treat null/undefined as an empty string', () => { @@ -79,6 +100,8 @@ describe('utils/packageHandle', () => { }) it('should return empty string if not enough values', () => { + console.log = jest.fn() + console.error = jest.fn() expect( packageHandle.execTemplate( { files: '<%= username %>/<%= directory %>' }, @@ -88,6 +111,12 @@ describe('utils/packageHandle', () => { }, ), ).toBe('') + expect(console.log).toHaveBeenCalledWith( + 'Template for default package name is invalid', + ) + expect((console.error as jest.Mock).mock.calls[0][0]).toMatchObject({ + message: 'directory is not defined', + }) }) }) }) From 33a76246a2bec7ed1d73a6d0e87e76466f930603 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 11:40:24 +0300 Subject: [PATCH 39/55] use caret for version --- catalog/app/utils/workflows.ts | 40 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index 7581bdbcf6..b034279649 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -152,23 +152,35 @@ const parseSuccessor = (url: string, successor: SuccessorYaml): Successor => ({ url, }) -function validateConfigVersion(objectVersion: WorkflowsVersion): undefined | Error[] { - const baseSchemaVersion = '<=1.1.0' - const catalogSchemaVersion = '<=1.0.0' +function validateConfigVersion( + objectVersion: string, + schemaVersion: string, +): undefined | Error { + if (semver.satisfies(schemaVersion, `^${objectVersion}`)) return undefined + + return new Error( + `Your config file version (${objectVersion}) is incompatible with the current version of the config schema (${schemaVersion})`, + ) +} - const { base: baseVersion, catalog: catalogVersion } = objectVersion +function validateConfigCompoundVersion( + objectCompoundVersion: WorkflowsVersion, +): undefined | Error[] { + const { base: baseVersion, catalog: catalogVersion } = objectCompoundVersion const errors = [] - if (!semver.satisfies(semver.coerce(baseVersion) || '0.0.0', baseSchemaVersion)) { - errors.push(new Error(`Catalog supports ${baseSchemaVersion} version of base Schema`)) + + const invalidBaseVersion = validateConfigVersion(baseVersion, '1.1.0') + if (invalidBaseVersion) { + errors.push(invalidBaseVersion) } - if ( - catalogVersion && - !semver.satisfies(semver.coerce(catalogVersion) || '0.0.0', catalogSchemaVersion) - ) { - errors.push( - new Error(`Catalog supports ${catalogSchemaVersion} version of catalog Schema`), - ) + + if (catalogVersion) { + const invalidCatalogVersion = validateConfigVersion(catalogVersion, '1.0.0') + if (invalidCatalogVersion) { + errors.push(invalidCatalogVersion) + } } + return errors.length ? errors : undefined } @@ -178,7 +190,7 @@ function validateConfig(data: unknown): asserts data is WorkflowsYaml { ? makeSchemaValidator(workflowsCatalogConfigSchema, [workflowsConfigSchema]) : makeSchemaValidator(workflowsConfigSchema) - const versionErrors = validateConfigVersion( + const versionErrors = validateConfigCompoundVersion( typeof objectVersion === 'string' ? { base: objectVersion } : objectVersion, ) if (versionErrors) From 9a52687445550421eda6e2155b1511c369892381 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 11:51:48 +0300 Subject: [PATCH 40/55] disable entries validation for package from direcotry --- .../Bucket/PackageDirectoryDialog.tsx | 44 ++----------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 68e614be4e..ff7a25daf9 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -1,4 +1,3 @@ -import type { ErrorObject } from 'ajv' import cx from 'classnames' import * as FF from 'final-form' import { basename } from 'path' @@ -16,7 +15,6 @@ import * as validators from 'utils/validators' import type * as workflows from 'utils/workflows' import * as PD from './PackageDialog' -import MetaInputErrorHelper from './PackageDialog/MetaInputErrorHelper' import * as requests from './requests' const prepareEntries = (entries: PD.FilesSelectorState, path: string) => { @@ -55,14 +53,6 @@ const useStyles = M.makeStyles((t) => ({ files: { height: '100%', }, - filesWithError: { - height: `calc(90% - ${t.spacing()}px)`, - }, - filesError: { - marginTop: t.spacing(), - maxHeight: t.spacing(9), - overflowY: 'auto', - }, form: { height: '100%', }, @@ -111,13 +101,8 @@ function DialogForm({ const [nameWarning, setNameWarning] = React.useState('') const [metaHeight, setMetaHeight] = React.useState(0) const validateWorkflow = PD.useWorkflowValidator(workflowsConfig) - const validateEntries = PD.useEntriesValidator(selectedWorkflow) const classes = useStyles() - const [entriesError, setEntriesError] = React.useState<(Error | ErrorObject)[] | null>( - null, - ) - const createPackage = requests.useWrapPackage() const dialogContentClasses = PD.useContentStyles({ metaHeight }) @@ -136,20 +121,6 @@ function DialogForm({ // eslint-disable-next-line consistent-return }) => { try { - const entries = filesValue - .filter((f) => f.selected) - .map((f) => ({ - logical_key: f.name, - size: f.size, - })) - const error = await validateEntries(entries) - if (error) { - setEntriesError(error) - return { - files: 'schema', - } - } - const res = await createPackage( { ...values, @@ -171,7 +142,7 @@ function DialogForm({ return { [FF.FORM_ERROR]: errorMessage || PD.ERROR_MESSAGES.MANIFEST } } }, - [bucket, successor, createPackage, setSuccess, schema, path, validateEntries], + [bucket, successor, createPackage, setSuccess, schema, path], ) const initialFiles: PD.FilesSelectorState = React.useMemo( @@ -215,13 +186,11 @@ function DialogForm({ const [editorElement, setEditorElement] = React.useState(null) const onFormChange = React.useCallback( - async ({ dirtyFields, values }) => { + async ({ values }) => { if (document.body.contains(editorElement)) { setMetaHeight(editorElement!.clientHeight) } - if (dirtyFields?.files) setEntriesError(null) - handleNameChange(values.name) }, [editorElement, handleNameChange, setMetaHeight], @@ -348,9 +317,7 @@ function DialogForm({ - - From 20eb08f14c1b309abdc0b97564edd2a5d593c4ae Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Fri, 8 Oct 2021 12:03:04 +0300 Subject: [PATCH 41/55] validate s3 files too --- .../app/containers/Bucket/PackageDialog/PackageCreationForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 6e54d2897e..6185d2439a 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -183,7 +183,7 @@ export function PackageCreationForm({ return !e || e.hash !== file.hash.value }) - const entries = toUpload.map(({ file, path }) => ({ + const entries = [...addedS3Entries, ...toUpload].map(({ file, path }) => ({ logical_key: path, size: file.size, })) From dabd0123cabe0352531a66051df443fdab43833b Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 11 Oct 2021 08:14:51 +0300 Subject: [PATCH 42/55] update version Schema --- shared/schemas/workflows-config-1.1.0.json | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/shared/schemas/workflows-config-1.1.0.json b/shared/schemas/workflows-config-1.1.0.json index 3cade0250f..5f22da6140 100644 --- a/shared/schemas/workflows-config-1.1.0.json +++ b/shared/schemas/workflows-config-1.1.0.json @@ -4,9 +4,37 @@ "type": "object", "required": ["version", "workflows"], "additionalProperties": false, + "definitions": { + "Version": { + "type": "string", + "pattern": "^[1-9][0-9]*(\\.(0|[1-9][0-9]*)){0,2}$", + "examples": ["1", "1.0"] + }, + "CompoundVersion": { + "type": "object", + "additionalProperties": false, + "required": ["base"], + "properties": { + "base": { + "$ref": "#/definitions/Version" + }, + "catalog": { + "$ref": "#/definitions/Version" + } + } + } + }, "properties": { "catalog": true, "version": { + "oneOf": [ + { + "$ref": "#/definitions/Version" + }, + { + "$ref": "#/definitions/CompoundVersion" + } + ], "type": "string", "pattern": "^[1-9][0-9]*(\\.[0-9]+)?( catalog:[1-9][0-9]*(\\.[0-9]+)?)*$" }, From 660dea72e0e1e66be5c1d9f8b3fcc913f01c1e40 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 11 Oct 2021 08:19:10 +0300 Subject: [PATCH 43/55] use workflow value intsead of workflow getter --- catalog/app/containers/Bucket/PackageCopyDialog.js | 4 ++-- .../Bucket/PackageDialog/PackageCreationForm.tsx | 4 ++-- .../app/containers/Bucket/PackageDialog/PackageDialog.tsx | 8 ++++---- catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index 61e0508ade..d8350c2bd3 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -156,7 +156,7 @@ function DialogForm({ [editorElement, handleNameChange, selectedWorkflow, setMetaHeight, setWorkflow], ) - const getWorkflow = React.useCallback( + const workflowTemplate = React.useCallback( () => selectedWorkflow || workflowsConfig, [selectedWorkflow, workflowsConfig], ) @@ -221,7 +221,7 @@ function DialogForm({ selectedWorkflow || workflowsConfig, [selectedWorkflow, workflowsConfig], ) @@ -388,7 +388,7 @@ export function PackageCreationForm({ directory?: string meta: RF.FieldMetaState - getWorkflow: () => { packageName: packageHandleUtils.NameTemplates } validating: boolean + workflow: { packageName: packageHandleUtils.NameTemplates } } type PackageNameInputProps = PackageNameInputOwnProps & @@ -260,7 +260,7 @@ export function PackageNameInput({ errors, input: { value, onChange }, meta, - getWorkflow, + workflow, directory, validating, ...rest @@ -293,7 +293,7 @@ export function PackageNameInput({ React.useEffect(() => { if (modified) return - const packageName = getDefaultPackageName(getWorkflow(), { + const packageName = getDefaultPackageName(workflow, { username, directory, }) @@ -304,7 +304,7 @@ export function PackageNameInput({ value: packageName, }, }) - }, [directory, getWorkflow, modified, onChange, username]) + }, [directory, workflow, modified, onChange, username]) return } diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index ff7a25daf9..81f132c5f5 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -202,7 +202,7 @@ function DialogForm({ } }, [editorElement, setMetaHeight]) - const getWorkflow = React.useCallback( + const workflowTemplate = React.useCallback( () => selectedWorkflow || workflowsConfig, [selectedWorkflow, workflowsConfig], ) @@ -268,7 +268,7 @@ function DialogForm({ Date: Mon, 11 Oct 2021 13:21:58 +0300 Subject: [PATCH 44/55] Apply suggestions from code review Co-authored-by: Alexei Mochalov --- catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 2 +- shared/schemas/workflows-config-1.1.0.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx index 81f132c5f5..97f88fee95 100644 --- a/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDirectoryDialog.tsx @@ -317,7 +317,7 @@ function DialogForm({ Date: Mon, 11 Oct 2021 14:09:34 +0300 Subject: [PATCH 45/55] add workflow calculation inline --- catalog/app/containers/Bucket/PackageCopyDialog.js | 7 +------ .../Bucket/PackageDialog/PackageCreationForm.tsx | 7 +------ catalog/app/containers/Bucket/PackageDirectoryDialog.tsx | 8 +------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCopyDialog.js b/catalog/app/containers/Bucket/PackageCopyDialog.js index d8350c2bd3..9cf833cc42 100644 --- a/catalog/app/containers/Bucket/PackageCopyDialog.js +++ b/catalog/app/containers/Bucket/PackageCopyDialog.js @@ -156,11 +156,6 @@ function DialogForm({ [editorElement, handleNameChange, selectedWorkflow, setMetaHeight, setWorkflow], ) - const workflowTemplate = React.useCallback( - () => selectedWorkflow || workflowsConfig, - [selectedWorkflow, workflowsConfig], - ) - React.useEffect(() => { if (document.body.contains(editorElement)) { setMetaHeight(editorElement.clientHeight) @@ -221,7 +216,7 @@ function DialogForm({ selectedWorkflow || workflowsConfig, - [selectedWorkflow, workflowsConfig], - ) - return ( selectedWorkflow || workflowsConfig, - [selectedWorkflow, workflowsConfig], - ) - return ( Date: Mon, 11 Oct 2021 15:03:12 +0300 Subject: [PATCH 46/55] remove catalog if there is no catalog version in workflows/config.yaml --- catalog/app/utils/workflows.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/catalog/app/utils/workflows.ts b/catalog/app/utils/workflows.ts index b034279649..78207e0316 100644 --- a/catalog/app/utils/workflows.ts +++ b/catalog/app/utils/workflows.ts @@ -18,7 +18,7 @@ interface WorkflowsVersion { interface WorkflowsYaml { default_workflow?: string is_workflow_required?: boolean - catalog: { + catalog?: { package_handle?: packageHandleUtils.NameTemplates } schemas?: Record @@ -34,7 +34,7 @@ interface WorkflowYaml { is_message_required?: boolean metadata_schema?: string name: string - catalog: { + catalog?: { package_handle?: packageHandleUtils.NameTemplates } } @@ -200,11 +200,22 @@ function validateConfig(data: unknown): asserts data is WorkflowsYaml { if (errors.length) throw new bucketErrors.WorkflowsConfigInvalid({ errors }) } +function prepareData(data: unknown): WorkflowsYaml { + validateConfig(data) + + if ((data.version as WorkflowsVersion).catalog) return data + + const removeCatalog = R.dissoc('catalog') + return removeCatalog( + R.over(R.lensProp('workflows'), R.mapObjIndexed(removeCatalog), data), + ) +} + export function parse(workflowsYaml: string): WorkflowsConfig { - const data = yaml(workflowsYaml) - if (!data) return emptyConfig + const rawData = yaml(workflowsYaml) + if (!rawData) return emptyConfig - validateConfig(data) + const data = prepareData(rawData) const { workflows } = data const workflowsList = Object.keys(workflows).map((slug) => From ffae8c58dd971e72cb1e7a78df0b22e89f49237c Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 11 Oct 2021 16:08:27 +0300 Subject: [PATCH 47/55] use all files state to calc actual files --- .../Bucket/PackageDialog/PackageCreationForm.tsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 6b22e4d20c..1f765db5c2 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -183,10 +183,16 @@ export function PackageCreationForm({ return !e || e.hash !== file.hash.value }) - const entries = [...addedS3Entries, ...toUpload].map(({ file, path }) => ({ - logical_key: path, - size: file.size, - })) + const entries = FP.function.pipe( + R.mergeLeft(files.added, files.existing), + R.omit(Object.keys(files.deleted)), + Object.entries, + R.map(([path, file]) => ({ + logical_key: path, + size: file.size, + })), + ) + const error = await validateEntries(entries) if (error && error.length) { setEntriesError(error) From 6087f5601ed831101138035a6af8b5a50226ad1e Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Mon, 11 Oct 2021 16:46:46 +0300 Subject: [PATCH 48/55] fix schema --- shared/schemas/workflows-config-1.1.0.json | 4 +--- shared/schemas/workflows-config_catalog-1.0.0.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/shared/schemas/workflows-config-1.1.0.json b/shared/schemas/workflows-config-1.1.0.json index 99adff4a3d..8876224cc6 100644 --- a/shared/schemas/workflows-config-1.1.0.json +++ b/shared/schemas/workflows-config-1.1.0.json @@ -34,9 +34,7 @@ { "$ref": "#/definitions/CompoundVersion" } - ], - "type": "string", - "pattern": "^[1-9][0-9]*(\\.[0-9]+)?( catalog:[1-9][0-9]*(\\.[0-9]+)?)*$" + ] }, "is_workflow_required": { "type": "boolean", diff --git a/shared/schemas/workflows-config_catalog-1.0.0.json b/shared/schemas/workflows-config_catalog-1.0.0.json index 3e21b8211f..41ac257d68 100644 --- a/shared/schemas/workflows-config_catalog-1.0.0.json +++ b/shared/schemas/workflows-config_catalog-1.0.0.json @@ -56,7 +56,7 @@ }, "allOf": [ { - "$ref": "https://schemas.quiltdata.com/workflows-config-1.1.0.json" + "$ref": "https://schemas.quiltdata.com/workflows-config-1.1.0" }, { "$ref": "#/definitions/CatalogSettings" From 607df74278945cfadc068772ec90039f97f507e5 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 09:42:37 +0300 Subject: [PATCH 49/55] fix mistypings --- catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index 4d67e34886..3b35a4e86d 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -818,9 +818,10 @@ const getDefaultPackageName = ( workflow: { packageName: packageHandleUtils.NameTemplates }, { directory, username }: { directory?: string; username: string }, ) => - directory + typeof directory === 'string' ? packageHandleUtils.execTemplate(workflow?.packageName, 'files', { directory: basename(directory), + username: s3paths.ensureNoSlash(getUsernamePrefix(username)), }) : packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { username: s3paths.ensureNoSlash(getUsernamePrefix(username)), From 89fae6e2627a3e37052d14a80041038ce15d4097 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 10:31:23 +0300 Subject: [PATCH 50/55] catch Ajv errors --- catalog/app/utils/json-schema/json-schema.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/catalog/app/utils/json-schema/json-schema.ts b/catalog/app/utils/json-schema/json-schema.ts index 71acd3f7c3..e13502575c 100644 --- a/catalog/app/utils/json-schema/json-schema.ts +++ b/catalog/app/utils/json-schema/json-schema.ts @@ -147,11 +147,13 @@ export function makeSchemaValidator( useDefaults: true, ...ajvOptions, } - const ajv = new Ajv(options) - addFormats(ajv, ['date', 'regex', 'uri']) - ajv.addKeyword('dateformat') try { + const ajv = new Ajv(options) + addFormats(ajv, ['date', 'regex', 'uri']) + ajv.addKeyword('dateformat') + + // TODO: fail early, return Error instead of callback if (!$id) return () => [new Error('$id is not provided')] return (obj: any): ErrorObject[] => { @@ -160,6 +162,7 @@ export function makeSchemaValidator( return ajv.errors || [] } } catch (e) { + // TODO: fail early if Ajv options are incorrect, return Error instead of callback // TODO: add custom errors return () => (e instanceof Error ? [e] : []) as Error[] } From 18b61eb4b1b133e33cc8a9aa939988493208efea Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 11:30:04 +0300 Subject: [PATCH 51/55] dont rewrite existing name --- .../containers/Bucket/PackageCreateDialog.tsx | 6 ----- .../Bucket/PackageDialog/PackageDialog.tsx | 24 +++++++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageCreateDialog.tsx b/catalog/app/containers/Bucket/PackageCreateDialog.tsx index 02b0e611a6..c7ff777829 100644 --- a/catalog/app/containers/Bucket/PackageCreateDialog.tsx +++ b/catalog/app/containers/Bucket/PackageCreateDialog.tsx @@ -1,8 +1,6 @@ import * as R from 'ramda' import * as React from 'react' -import * as redux from 'react-redux' -import * as authSelectors from 'containers/Auth/selectors' import * as AWS from 'utils/AWS' import AsyncResult from 'utils/AsyncResult' import * as BucketPreferences from 'utils/BucketPreferences' @@ -40,15 +38,11 @@ export function usePackageCreateDialog({ _: R.identity, }) - const username = redux.useSelector(authSelectors.username) - const usernamePrefix = React.useMemo(() => PD.getUsernamePrefix(username), [username]) - return PD.usePackageCreationDialog({ bucket, data, delayHashing: true, disableStateDisplay: true, - name: usernamePrefix, onExited, ui: { successTitle: 'Package created', diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index 3b35a4e86d..a6dc2e081e 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -268,7 +268,7 @@ export function PackageNameInput({ const readyForValidation = (value && meta.modified) || meta.submitFailed const errorCode = readyForValidation && meta.error const error = errorCode ? errors[errorCode] || errorCode : '' - const [modified, setModified] = React.useState(meta.modified) + const [modified, setModified] = React.useState(!!(meta.modified || value)) const handleChange = React.useCallback( (event) => { setModified(true) @@ -817,15 +817,19 @@ export function getUsernamePrefix(username?: string | null) { const getDefaultPackageName = ( workflow: { packageName: packageHandleUtils.NameTemplates }, { directory, username }: { directory?: string; username: string }, -) => - typeof directory === 'string' - ? packageHandleUtils.execTemplate(workflow?.packageName, 'files', { - directory: basename(directory), - username: s3paths.ensureNoSlash(getUsernamePrefix(username)), - }) - : packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { - username: s3paths.ensureNoSlash(getUsernamePrefix(username)), - }) +) => { + const usernamePrefix = getUsernamePrefix(username) + const templateBasedName = + typeof directory === 'string' + ? packageHandleUtils.execTemplate(workflow?.packageName, 'files', { + directory: basename(directory), + username: s3paths.ensureNoSlash(usernamePrefix), + }) + : packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { + username: s3paths.ensureNoSlash(usernamePrefix), + }) + return templateBasedName || usernamePrefix +} const usePackageNameWarningStyles = M.makeStyles({ root: { From 1e648833c1daa2a8287a2691284bce13e7d4c390 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 12:14:30 +0300 Subject: [PATCH 52/55] adjust unit-test --- catalog/app/utils/json-schema/json-schema.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalog/app/utils/json-schema/json-schema.spec.ts b/catalog/app/utils/json-schema/json-schema.spec.ts index ce4f55cc42..f4d9023847 100644 --- a/catalog/app/utils/json-schema/json-schema.spec.ts +++ b/catalog/app/utils/json-schema/json-schema.spec.ts @@ -20,10 +20,10 @@ describe('utils/json-schema', () => { }) }) - it('should throw an error when schema is incorrect', () => { - expect(() => { - makeSchemaValidator(incorrect.schema) - }).toThrowError(/schema is invalid/) + it('should return an error when schema is incorrect', () => { + expect(makeSchemaValidator(incorrect.schema)()[0].message).toMatch( + /schema is invalid/, + ) }) describe('validator for Schema with enum types', () => { From a5b71c7a5f88b52a3bcf0d2297d6f3b2e0ca9be8 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 12:25:42 +0300 Subject: [PATCH 53/55] divide empty string made on purpose and lack of value --- .../app/containers/Bucket/PackageDialog/PackageDialog.tsx | 2 +- catalog/app/utils/packageHandle.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx index a6dc2e081e..8c4bd946b2 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx @@ -828,7 +828,7 @@ const getDefaultPackageName = ( : packageHandleUtils.execTemplate(workflow?.packageName, 'packages', { username: s3paths.ensureNoSlash(usernamePrefix), }) - return templateBasedName || usernamePrefix + return typeof templateBasedName === 'string' ? templateBasedName : usernamePrefix } const usePackageNameWarningStyles = M.makeStyles({ diff --git a/catalog/app/utils/packageHandle.ts b/catalog/app/utils/packageHandle.ts index 151adab43c..55d37d8cef 100644 --- a/catalog/app/utils/packageHandle.ts +++ b/catalog/app/utils/packageHandle.ts @@ -36,7 +36,7 @@ export function execTemplate( templatesDict: NameTemplates, context: Context, options?: Options, -): string { - if (!templatesDict) return '' - return execTemplateItem(templatesDict[context] || '', options) || '' +): string | null { + if (!templatesDict || !templatesDict[context]) return null + return execTemplateItem(templatesDict[context] || '', options) } From fb51a257f9e80aad5935289d487dd7ad2aa729d4 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 14:31:07 +0300 Subject: [PATCH 54/55] fix unit test --- catalog/app/utils/packageHandle.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/app/utils/packageHandle.spec.ts b/catalog/app/utils/packageHandle.spec.ts index 9f6e864c4d..ab092cb268 100644 --- a/catalog/app/utils/packageHandle.spec.ts +++ b/catalog/app/utils/packageHandle.spec.ts @@ -110,7 +110,7 @@ describe('utils/packageHandle', () => { username: 'fiskus', }, ), - ).toBe('') + ).toBe(null) expect(console.log).toHaveBeenCalledWith( 'Template for default package name is invalid', ) From 1ea1d7fae274a07d8a7fb4dc3bed29c828124f67 Mon Sep 17 00:00:00 2001 From: Maxim Chervonny Date: Thu, 14 Oct 2021 14:57:07 +0300 Subject: [PATCH 55/55] exporting and dont use exported is confusing --- .../Bucket/PackageDialog/PackageCreationForm.tsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx index 1f765db5c2..019661dd44 100644 --- a/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx +++ b/catalog/app/containers/Bucket/PackageDialog/PackageCreationForm.tsx @@ -99,7 +99,7 @@ interface PackageCreationFormProps { } } -export function PackageCreationForm({ +function PackageCreationForm({ bucket, close, initial, @@ -510,7 +510,7 @@ export function PackageCreationForm({ ) } -export const PackageCreationDialogState = tagged.create( +const PackageCreationDialogState = tagged.create( 'app/containers/Bucket/PackageDialog/PackageCreationForm:DialogState' as const, { Closed: () => {}, @@ -526,9 +526,7 @@ export const PackageCreationDialogState = tagged.create( ) // eslint-disable-next-line @typescript-eslint/no-redeclare -export type PackageCreationDialogState = tagged.InstanceOf< - typeof PackageCreationDialogState -> +type PackageCreationDialogState = tagged.InstanceOf interface UsePackageCreationDialogProps { bucket: string