From 01745d592122e3da03ee571bdda82c181e46e64c Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 19 Feb 2021 14:48:57 -0500 Subject: [PATCH 01/35] Update lock-files --- lerna-package-lock.json | 48 +-- server/routerlicious/lerna-package-lock.json | 281 ++++++++--------- server/routerlicious/package-lock.json | 300 +++++++++++-------- 3 files changed, 327 insertions(+), 302 deletions(-) diff --git a/lerna-package-lock.json b/lerna-package-lock.json index 033a62794e2e..b841b316c605 100644 --- a/lerna-package-lock.json +++ b/lerna-package-lock.json @@ -4342,9 +4342,9 @@ }, "dependencies": { "@types/node": { - "version": "10.17.53", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.53.tgz", - "integrity": "sha512-q1igVlMUU+10kzjxNlcLDH7gekuvFK1nevnp7MAyc6sqvK5siWSS37EuvKX9fM8d49SBcoP0iP9tqVHmdAjNhQ==" + "version": "10.17.54", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.54.tgz", + "integrity": "sha512-c8Lm7+hXdSPmWH4B9z/P/xIXhFK3mCQin4yCYMd2p1qpMG5AfgyJuYZ+3q2dT7qLiMMMGMd5dnkFpdqJARlvtQ==" } } }, @@ -4486,9 +4486,9 @@ }, "dependencies": { "@types/node": { - "version": "10.17.53", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.53.tgz", - "integrity": "sha512-q1igVlMUU+10kzjxNlcLDH7gekuvFK1nevnp7MAyc6sqvK5siWSS37EuvKX9fM8d49SBcoP0iP9tqVHmdAjNhQ==" + "version": "10.17.54", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.54.tgz", + "integrity": "sha512-c8Lm7+hXdSPmWH4B9z/P/xIXhFK3mCQin4yCYMd2p1qpMG5AfgyJuYZ+3q2dT7qLiMMMGMd5dnkFpdqJARlvtQ==" } } }, @@ -4662,9 +4662,9 @@ } }, "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" }, "ansi-regex": { "version": "2.1.1", @@ -4941,9 +4941,9 @@ } }, "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" } } }, @@ -31259,9 +31259,9 @@ } }, "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" }, "ansi-regex": { "version": "2.1.1", @@ -33094,9 +33094,9 @@ }, "dependencies": { "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" } } }, @@ -33119,9 +33119,9 @@ }, "dependencies": { "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" }, "jwt-decode": { "version": "3.1.2", @@ -33146,9 +33146,9 @@ }, "dependencies": { "@types/node": { - "version": "12.20.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.3.tgz", - "integrity": "sha512-63cSd8J30Sr4/aFKKfDmCEM4GMH3W2efWT0Ii/B+Ohm3id0TU2xPEBFktiq3nXCZcN6VwVvpyv75I4zTP7YO/w==" + "version": "12.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.4.tgz", + "integrity": "sha512-xRCgeE0Q4pT5UZ189TJ3SpYuX/QGl6QIAOAIeDSbAVAd2gX1NxSZup4jNVK7cxIeP8KDSbJgcckun495isP1jQ==" } } }, diff --git a/server/routerlicious/lerna-package-lock.json b/server/routerlicious/lerna-package-lock.json index 9098396c387e..b5a863154c07 100644 --- a/server/routerlicious/lerna-package-lock.json +++ b/server/routerlicious/lerna-package-lock.json @@ -549,9 +549,9 @@ "integrity": "sha512-YnlNOwp6+eN0L5gb2UXQQogm8Z59v+cQuUGbY87I31iTMv0N3ZY5DEHiUiQTLFgr3ccD4CsjpUOq8K1/7DO6tA==" }, "@fluidframework/build-tools": { - "version": "0.2.14220", - "resolved": "https://registry.npmjs.org/@fluidframework/build-tools/-/build-tools-0.2.14220.tgz", - "integrity": "sha512-iqJj+lgK2Yfxe85BJm60Km/fI39iVum042YFU/4hDWzPjv5iNGGQB5vG5ObX95fyk5vUMMpNHG39dTdLj1X5wg==", + "version": "0.2.16336", + "resolved": "https://registry.npmjs.org/@fluidframework/build-tools/-/build-tools-0.2.16336.tgz", + "integrity": "sha512-DzjNE1dIftNJgMlqTByIkgegwiA5sxS5nzdQG+i00oQnW2xOk6a8UInDABdvjuWaYCN/2Pd3BLzxoaN4hYK5eQ==", "dev": true, "requires": { "@fluidframework/bundle-size-tools": "^0.0.8505", @@ -640,9 +640,9 @@ }, "dependencies": { "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.9.tgz", + "integrity": "sha512-kdMjTiekY+z/ubJCATUPlRDl39vXYiMV9iyeMuEuXZh2we6zz80uovNN2WlAxmmdE/Z/YQe+EbOEXB5RHEED3w==", "dev": true } } @@ -2008,15 +2008,15 @@ }, "dependencies": { "@oclif/plugin-help": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-3.2.1.tgz", - "integrity": "sha512-vq7rn16TrQmjX3Al/k1Z5iBZWZ3HE8fDXs52OmDJmmTqryPSNvURH9WCAsqr0PODYCSR17Hy1VTzS0x7vVVLEQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-3.2.2.tgz", + "integrity": "sha512-SPZ8U8PBYK0n4srFjCLedk0jWU4QlxgEYLCXIBShJgOwPhTTQknkUlsEwaMIevvCU4iCQZhfMX+D8Pz5GZjFgA==", "dev": true, "requires": { "@oclif/command": "^1.5.20", "@oclif/config": "^1.15.1", "@oclif/errors": "^1.2.2", - "chalk": "^2.4.1", + "chalk": "^4.1.0", "indent-string": "^4.0.0", "lodash.template": "^4.4.0", "string-width": "^4.2.0", @@ -2031,12 +2031,52 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -2072,6 +2112,15 @@ "ansi-regex": "^5.0.0" } }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, "wrap-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-4.0.0.tgz", @@ -2089,6 +2138,30 @@ "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", "dev": true }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", @@ -2278,15 +2351,6 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, "clean-stack": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-3.0.1.tgz", @@ -2296,27 +2360,6 @@ "escape-string-regexp": "4.0.0" } }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true - }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -2334,23 +2377,6 @@ "universalify": "^0.1.0" } }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true - }, - "string-width": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", - "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", - "dev": true, - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" - } - }, "strip-ansi": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", @@ -2359,17 +2385,6 @@ "requires": { "ansi-regex": "^5.0.0" } - }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } } } }, @@ -6578,9 +6593,9 @@ "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=" }, "danger": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/danger/-/danger-10.6.0.tgz", - "integrity": "sha512-cN9skuxtkgfnV4CQpF20L3Yt3VhMlBvvWAoh2QashDepI420gwcxoCRpxdCF7GP3aVJuh5LuI9ObrDvAxpnCFA==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/danger/-/danger-10.6.2.tgz", + "integrity": "sha512-EVMone2dJsFzm7VbSspIsV9mOFIjLVdW7WxSO2PMnOCYYoDRKPT3mqgE8pfMWg0xclPjtfJJ7k3AZGirvNWWHQ==", "dev": true, "requires": { "@babel/polyfill": "^7.2.5", @@ -8485,6 +8500,12 @@ } } }, + "filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha1-mzERErxsYSehbgFsbF1/GeCAXFs=", + "dev": true + }, "finalhandler": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", @@ -10970,9 +10991,9 @@ } }, "jszip": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.5.0.tgz", - "integrity": "sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", "dev": true, "requires": { "lie": "~3.3.0", @@ -14759,12 +14780,13 @@ "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "query-string": { - "version": "6.13.8", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.13.8.tgz", - "integrity": "sha512-jxJzQI2edQPE/NPUOusNjO/ZOGqr1o2OBa/3M00fU76FsLXDVbJDv/p7ng5OdQyorKrkRz1oqfwmbe5MAMePQg==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.0.tgz", + "integrity": "sha512-In3o+lUxlgejoVJgwEdYtdxrmlL0cQWJXj0+kkI7RWVo7hg5AhFtybeKlC9Dpgbr8eOC4ydpEh8017WwyfzqVQ==", "dev": true, "requires": { "decode-uri-component": "^0.2.0", + "filter-obj": "^1.1.0", "split-on-first": "^1.0.0", "strict-uri-encode": "^2.0.0" } @@ -15208,14 +15230,14 @@ } }, "replace-in-file": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.1.0.tgz", - "integrity": "sha512-URzjyF3nucvejuY13HFd7O+Q6tFJRLKGHLYVvSh+LiZj3gFXzSYGnIkQflnJJulCAI2/RTZaZkpOtdVdW0EhQA==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.2.0.tgz", + "integrity": "sha512-Im2AF9G/qgkYneOc9QwWwUS/efyyonTUBvzXS2VXuxPawE5yQIjT/e6x4CTijO0Quq48lfAujuo+S89RR2TP2Q==", "dev": true, "requires": { - "chalk": "^4.0.0", + "chalk": "^4.1.0", "glob": "^7.1.6", - "yargs": "^15.3.1" + "yargs": "^16.2.0" }, "dependencies": { "ansi-regex": { @@ -15244,14 +15266,14 @@ } }, "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" + "wrap-ansi": "^7.0.0" } }, "color-convert": { @@ -15275,16 +15297,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -15297,30 +15309,6 @@ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true - }, "string-width": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", @@ -15350,45 +15338,32 @@ "has-flag": "^4.0.0" } }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } + "y18n": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", + "dev": true }, "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" } }, "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } + "version": "20.2.5", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.5.tgz", + "integrity": "sha512-jYRGS3zWy20NtDtK2kBgo/TlAoy5YUuhD9/LZ7z7W4j1Fdw2cqD0xEEclf8fxc8xjD6X5Qr+qQQwCEsP8iRiYg==", + "dev": true } } }, @@ -17873,9 +17848,9 @@ } }, "typed-rest-client": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.0.tgz", - "integrity": "sha512-Nu1MrdH6ECrRW5gHoRAdubgCs4oH6q5/J76jsEC8bVDfvVoVPkigukPalhMHPwb7ZvpsZqPptd5zpt/QdtrdBw==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.1.tgz", + "integrity": "sha512-7JbJFBZZuu3G64u6ksklN1xtVGfqBKiR5MQoTe5oLTi68OyB6pRuuIQCllfK/BdGjQtZYp62rgUOnEYDz4e9Xg==", "dev": true, "requires": { "qs": "^6.9.1", diff --git a/server/routerlicious/package-lock.json b/server/routerlicious/package-lock.json index 0631993aaf00..d2bc7fd7dc1a 100644 --- a/server/routerlicious/package-lock.json +++ b/server/routerlicious/package-lock.json @@ -386,9 +386,9 @@ } }, "@fluidframework/build-tools": { - "version": "0.2.14220", - "resolved": "https://registry.npmjs.org/@fluidframework/build-tools/-/build-tools-0.2.14220.tgz", - "integrity": "sha512-iqJj+lgK2Yfxe85BJm60Km/fI39iVum042YFU/4hDWzPjv5iNGGQB5vG5ObX95fyk5vUMMpNHG39dTdLj1X5wg==", + "version": "0.2.16336", + "resolved": "https://registry.npmjs.org/@fluidframework/build-tools/-/build-tools-0.2.16336.tgz", + "integrity": "sha512-DzjNE1dIftNJgMlqTByIkgegwiA5sxS5nzdQG+i00oQnW2xOk6a8UInDABdvjuWaYCN/2Pd3BLzxoaN4hYK5eQ==", "dev": true, "requires": { "@fluidframework/bundle-size-tools": "^0.0.8505", @@ -440,9 +440,9 @@ } }, "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.9.tgz", + "integrity": "sha512-kdMjTiekY+z/ubJCATUPlRDl39vXYiMV9iyeMuEuXZh2we6zz80uovNN2WlAxmmdE/Z/YQe+EbOEXB5RHEED3w==", "dev": true }, "universalify": { @@ -468,9 +468,9 @@ }, "dependencies": { "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.9.tgz", + "integrity": "sha512-kdMjTiekY+z/ubJCATUPlRDl39vXYiMV9iyeMuEuXZh2we6zz80uovNN2WlAxmmdE/Z/YQe+EbOEXB5RHEED3w==", "dev": true } } @@ -1803,15 +1803,15 @@ }, "dependencies": { "@oclif/plugin-help": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-3.2.1.tgz", - "integrity": "sha512-vq7rn16TrQmjX3Al/k1Z5iBZWZ3HE8fDXs52OmDJmmTqryPSNvURH9WCAsqr0PODYCSR17Hy1VTzS0x7vVVLEQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@oclif/plugin-help/-/plugin-help-3.2.2.tgz", + "integrity": "sha512-SPZ8U8PBYK0n4srFjCLedk0jWU4QlxgEYLCXIBShJgOwPhTTQknkUlsEwaMIevvCU4iCQZhfMX+D8Pz5GZjFgA==", "dev": true, "requires": { "@oclif/command": "^1.5.20", "@oclif/config": "^1.15.1", "@oclif/errors": "^1.2.2", - "chalk": "^2.4.1", + "chalk": "^4.1.0", "indent-string": "^4.0.0", "lodash.template": "^4.4.0", "string-width": "^4.2.0", @@ -1826,12 +1826,52 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -1858,6 +1898,15 @@ "ansi-regex": "^5.0.0" } }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + }, "wrap-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-4.0.0.tgz", @@ -1875,6 +1924,30 @@ "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", "dev": true }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", @@ -4095,9 +4168,9 @@ "dev": true }, "danger": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/danger/-/danger-10.6.0.tgz", - "integrity": "sha512-cN9skuxtkgfnV4CQpF20L3Yt3VhMlBvvWAoh2QashDepI420gwcxoCRpxdCF7GP3aVJuh5LuI9ObrDvAxpnCFA==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/danger/-/danger-10.6.2.tgz", + "integrity": "sha512-EVMone2dJsFzm7VbSspIsV9mOFIjLVdW7WxSO2PMnOCYYoDRKPT3mqgE8pfMWg0xclPjtfJJ7k3AZGirvNWWHQ==", "dev": true, "requires": { "@babel/polyfill": "^7.2.5", @@ -4538,6 +4611,12 @@ "es6-promise": "^4.0.3" } }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -4807,9 +4886,9 @@ "dev": true }, "fastq": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.10.0.tgz", - "integrity": "sha512-NL2Qc5L3iQEsyYzweq7qfgy5OtXCmGzGvhElGEd/SoFWEMOEczNh5s5ocaF01HDetxz+p8ecjNPA6cZxxIHmzA==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.10.1.tgz", + "integrity": "sha512-AWuv6Ery3pM+dY7LYS8YIaCiQvUaos9OB1RyNgaOWnaX+Tik7Onvcsf8x8c+YtDeT0maYLniBip2hox5KtEXXA==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -4853,6 +4932,12 @@ } } }, + "filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha1-mzERErxsYSehbgFsbF1/GeCAXFs=", + "dev": true + }, "find-cache-dir": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", @@ -5242,9 +5327,9 @@ "dev": true }, "get-intrinsic": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.0.tgz", - "integrity": "sha512-M11rgtQp5GZMZzDL7jLTNxbDfurpzuau5uqRWDPvlHjfvg3TdScAZo96GLvhMjImrmR8uAt0FS2RLoMrfWGKlg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", "dev": true, "requires": { "function-bind": "^1.1.1", @@ -6506,14 +6591,14 @@ } }, "is-typed-array": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.4.tgz", - "integrity": "sha512-ILaRgn4zaSrVNXNGtON6iFNotXW3hAPF3+0fB1usg2jFlWqo5fEDdmJkz0zBfoi7Dgskr8Khi2xZ8cXqZEfXNA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.5.tgz", + "integrity": "sha512-S+GRDgJlR3PyEbsX/Fobd9cqpZBuvUS+8asRqYDMLCb2qMzt1oz5m5oxQCxOgUDxiWsOVNi4yaF+/uvdlHlYug==", "dev": true, "requires": { "available-typed-arrays": "^1.0.2", - "call-bind": "^1.0.0", - "es-abstract": "^1.18.0-next.1", + "call-bind": "^1.0.2", + "es-abstract": "^1.18.0-next.2", "foreach": "^2.0.5", "has-symbols": "^1.0.1" }, @@ -6541,17 +6626,18 @@ } }, "is-callable": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", - "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", + "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==", "dev": true }, "is-regex": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", - "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.2.tgz", + "integrity": "sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg==", "dev": true, "requires": { + "call-bind": "^1.0.2", "has-symbols": "^1.0.1" } }, @@ -6977,9 +7063,9 @@ } }, "jszip": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.5.0.tgz", - "integrity": "sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", "dev": true, "requires": { "lie": "~3.3.0", @@ -9063,16 +9149,23 @@ "dev": true }, "query-string": { - "version": "6.13.8", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.13.8.tgz", - "integrity": "sha512-jxJzQI2edQPE/NPUOusNjO/ZOGqr1o2OBa/3M00fU76FsLXDVbJDv/p7ng5OdQyorKrkRz1oqfwmbe5MAMePQg==", + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.14.0.tgz", + "integrity": "sha512-In3o+lUxlgejoVJgwEdYtdxrmlL0cQWJXj0+kkI7RWVo7hg5AhFtybeKlC9Dpgbr8eOC4ydpEh8017WwyfzqVQ==", "dev": true, "requires": { "decode-uri-component": "^0.2.0", + "filter-obj": "^1.1.0", "split-on-first": "^1.0.0", "strict-uri-encode": "^2.0.0" } }, + "queue-microtask": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.2.tgz", + "integrity": "sha512-dB15eXv3p2jDlbOiNLyMabYg1/sXvppd8DP2J3EOCQ0AkuSXCW2tP7mnVouVLJKgUMY6yP0kcQDVpLCN13h4Xg==", + "dev": true + }, "quick-lru": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", @@ -9306,14 +9399,14 @@ } }, "replace-in-file": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.1.0.tgz", - "integrity": "sha512-URzjyF3nucvejuY13HFd7O+Q6tFJRLKGHLYVvSh+LiZj3gFXzSYGnIkQflnJJulCAI2/RTZaZkpOtdVdW0EhQA==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/replace-in-file/-/replace-in-file-6.2.0.tgz", + "integrity": "sha512-Im2AF9G/qgkYneOc9QwWwUS/efyyonTUBvzXS2VXuxPawE5yQIjT/e6x4CTijO0Quq48lfAujuo+S89RR2TP2Q==", "dev": true, "requires": { - "chalk": "^4.0.0", + "chalk": "^4.1.0", "glob": "^7.1.6", - "yargs": "^15.3.1" + "yargs": "^16.2.0" }, "dependencies": { "ansi-regex": { @@ -9342,14 +9435,14 @@ } }, "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" + "wrap-ansi": "^7.0.0" } }, "color-convert": { @@ -9373,16 +9466,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -9395,30 +9478,6 @@ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true - }, "string-width": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", @@ -9448,45 +9507,32 @@ "has-flag": "^4.0.0" } }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } + "y18n": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", + "dev": true }, "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" } }, "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } + "version": "20.2.5", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.5.tgz", + "integrity": "sha512-jYRGS3zWy20NtDtK2kBgo/TlAoy5YUuhD9/LZ7z7W4j1Fdw2cqD0xEEclf8fxc8xjD6X5Qr+qQQwCEsP8iRiYg==", + "dev": true } } }, @@ -9618,10 +9664,13 @@ "dev": true }, "run-parallel": { - "version": "1.1.10", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.10.tgz", - "integrity": "sha512-zb/1OuZ6flOlH6tQyMPUrE3x3Ulxjlo9WIVXR4yVYi4H9UXQaeIsPbLn2R3O3vQCnDKkAl2qHiuocKKX4Tz/Sw==", - "dev": true + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "requires": { + "queue-microtask": "^1.2.2" + } }, "run-queue": { "version": "1.0.3", @@ -10717,9 +10766,9 @@ "dev": true }, "typed-rest-client": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.0.tgz", - "integrity": "sha512-Nu1MrdH6ECrRW5gHoRAdubgCs4oH6q5/J76jsEC8bVDfvVoVPkigukPalhMHPwb7ZvpsZqPptd5zpt/QdtrdBw==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.1.tgz", + "integrity": "sha512-7JbJFBZZuu3G64u6ksklN1xtVGfqBKiR5MQoTe5oLTi68OyB6pRuuIQCllfK/BdGjQtZYp62rgUOnEYDz4e9Xg==", "dev": true, "requires": { "qs": "^6.9.1", @@ -11059,17 +11108,18 @@ } }, "is-callable": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", - "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", + "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==", "dev": true }, "is-regex": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", - "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.2.tgz", + "integrity": "sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg==", "dev": true, "requires": { + "call-bind": "^1.0.2", "has-symbols": "^1.0.1" } }, From e94a8e20f23f1ceeeea773b07e4b50797988c797 Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 23 Feb 2021 17:53:24 -0500 Subject: [PATCH 02/35] Expose summarize function which returns tree --- .../container-runtime/src/containerRuntime.ts | 140 ++++++++++-------- .../container-runtime/src/summarizer.ts | 31 ++-- 2 files changed, 95 insertions(+), 76 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 046d030d54c4..242c905568cc 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -109,7 +109,7 @@ import { v4 as uuid } from "uuid"; import { ContainerFluidHandleContext } from "./containerHandleContext"; import { FluidDataStoreRegistry } from "./dataStoreRegistry"; import { debug } from "./debug"; -import { ISummarizerRuntime, ISummarizerInternalsProvider, Summarizer } from "./summarizer"; +import { ISummarizerRuntime, ISummarizerInternalsProvider, Summarizer, IGenerateSummaryOptions } from "./summarizer"; import { SummaryManager } from "./summaryManager"; import { analyzeTasks } from "./taskAnalyzer"; import { DeltaScheduler } from "./deltaScheduler"; @@ -1309,27 +1309,6 @@ export class ContainerRuntime extends TypedEventEmitter return this.context.submitSignalFn(envelope); } - /** - * Returns a summary of the runtime at the current sequence number. - * @param fullTree - true to bypass optimizations and force a full summary tree. - * @param trackState - This tells whether we should track state from this summary. - */ - private async summarize(fullTree: boolean = false, trackState: boolean = true): Promise { - const summarizeResult = await this.summarizerNode.summarize(fullTree, trackState); - assert(summarizeResult.summary.type === SummaryType.Tree, - "Container Runtime's summarize should always return a tree"); - return summarizeResult as IChannelSummarizeResult; - } - - private async summarizeInternal(fullTree: boolean, trackState: boolean): Promise { - const summarizeResult = await this.dataStores.summarize(fullTree, trackState); - this.addContainerBlobsToSummary(summarizeResult); - return { - ...summarizeResult, - id: "", - }; - } - public setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void { if (attachState === AttachState.Attaching) { assert(this.attachState === AttachState.Attaching, @@ -1356,17 +1335,77 @@ export class ContainerRuntime extends TypedEventEmitter return this.context.getAbsoluteUrl(relativeUrl); } - /** Implementation of ISummarizerInternalsProvider.generateSummary */ - public async generateSummary( - fullTree: boolean = false, - safe: boolean = false, + public async collectGarbage(logger: ITelemetryLogger) { + await PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => { + const gcStats: { totalGCNodes?: number; deletedGCNodes?: number } = {}; + try { + // Get the container's GC data and run GC on the reference graph in it. + const gcData = await this.dataStores.getGCData(this.runtimeOptions.runFullGC === true); + const { referencedNodeIds, deletedNodeIds } = runGarbageCollection( + gcData.gcNodes, [ "/" ], + this.logger, + ); + + // Update stats to be reported in the peformance event. + gcStats.deletedGCNodes = deletedNodeIds.length; + gcStats.totalGCNodes = referencedNodeIds.length + gcStats.deletedGCNodes; + + // Update our summarizer node's used routes. Updating used routes in summarizer node before + // summarizing is required and asserted by the the summarizer node. We are the root and are + // always referenced, so the used routes is only self-route (empty string). + this.summarizerNode.updateUsedRoutes([""]); + + // Remove this node's route ("/") and notify data stores of routes that are used in it. + const usedRoutes = referencedNodeIds.filter((id: string) => { return id !== "/"; }); + this.dataStores.updateUsedRoutes(usedRoutes); + } catch (error) { + event.cancel(gcStats, error); + throw error; + } + event.end(gcStats); + }); + } + + private async summarizeInternal(fullTree: boolean, trackState: boolean): Promise { + const summarizeResult = await this.dataStores.summarize(fullTree, trackState); + this.addContainerBlobsToSummary(summarizeResult); + return { + ...summarizeResult, + id: "", + }; + } + + /** + * Returns a summary of the runtime at the current sequence number. + */ + public async summarize(options: { + /** True to run garbage collection before summarizing */ + runGc: boolean, + /** True to generate the full tree with no handle reuse optimizations */ + fullTree: boolean, + /** True to track the state for this summary in the SummarizerNodes */ + trackState: boolean, + /** Logger to use for correlated summary events */ summaryLogger: ITelemetryLogger, - ): Promise { + }): Promise { + if (options.runGc) { + await this.collectGarbage(options.summaryLogger); + } + + const summarizeResult = await this.summarizerNode.summarize(options.fullTree, options.trackState); + assert(summarizeResult.summary.type === SummaryType.Tree, + "Container Runtime's summarize should always return a tree"); + + return summarizeResult as IChannelSummarizeResult; + } + + /** Implementation of ISummarizerInternalsProvider.generateSummary */ + public async generateSummary(options: IGenerateSummaryOptions): Promise { const summaryRefSeqNum = this.deltaManager.lastSequenceNumber; const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`; - this.summarizerNode.startSummary(summaryRefSeqNum, summaryLogger); + this.summarizerNode.startSummary(summaryRefSeqNum, options.summaryLogger); try { await this.deltaManager.inbound.pause(); @@ -1381,41 +1420,13 @@ export class ContainerRuntime extends TypedEventEmitter return { ...attemptData, reason: "disconnected" }; } - if (!this.runtimeOptions.disableGC) { - const perfEvent = PerformanceEvent.start(summaryLogger, { - eventName: "GarbageCollection", - }); - - const gcStats: { totalGCNodes?: number; deletedGCNodes?: number } = {}; - try { - // Get the container's GC data and run GC on the reference graph in it. - const gcData = await this.dataStores.getGCData(this.runtimeOptions.runFullGC === true); - const { referencedNodeIds, deletedNodeIds } = runGarbageCollection( - gcData.gcNodes, [ "/" ], - this.logger, - ); - - // Update stats to be reported in the peformance event. - gcStats.deletedGCNodes = deletedNodeIds.length; - gcStats.totalGCNodes = referencedNodeIds.length + gcStats.deletedGCNodes; - - // Update our summarizer node's used routes. Updating used routes in summarizer node before - // summarizing is required and asserted by the the summarizer node. We are the root and are - // always referenced, so the used routes is only self-route (empty string). - this.summarizerNode.updateUsedRoutes([""]); - - // Remove this node's route ("/") and notify data stores of routes that are used in it. - const usedRoutes = referencedNodeIds.filter((id: string) => { return id !== "/"; }); - this.dataStores.updateUsedRoutes(usedRoutes); - } catch (error) { - perfEvent.cancel(gcStats, error); - throw error; - } - perfEvent.end(gcStats); - } - const trace = Trace.start(); - const summarizeResult = await this.summarize(fullTree || safe, true /* trackState */); + const summarizeResult = await this.summarize({ + runGc: !this.runtimeOptions.disableGC, + fullTree: options.fullTree, + trackState: true, + summaryLogger: options.summaryLogger, + }); const generateData: IGeneratedSummaryData = { summaryStats: summarizeResult.stats, @@ -1437,13 +1448,12 @@ export class ContainerRuntime extends TypedEventEmitter summarizeResult.summary, this.latestSummaryAck); - // safe mode refreshes the latest summary ack - if (safe) { + if (options.refreshLatestAck) { const version = await this.getVersionFromStorage(this.id); await this.refreshLatestSummaryAck( undefined, version.id, - new ChildLogger(summaryLogger, undefined, { safeSummary: true }), + new ChildLogger(options.summaryLogger, undefined, { safeSummary: true }), version, ); } diff --git a/packages/runtime/container-runtime/src/summarizer.ts b/packages/runtime/container-runtime/src/summarizer.ts index 7d772479b792..2dd438796ed2 100644 --- a/packages/runtime/container-runtime/src/summarizer.ts +++ b/packages/runtime/container-runtime/src/summarizer.ts @@ -53,13 +53,18 @@ export interface IProvideSummarizer { readonly ISummarizer: ISummarizer; } +export interface IGenerateSummaryOptions { + /** True to generate the full tree with no handle reuse optimizations */ + fullTree: boolean, + /** True to ask the server what the latest summary is first */ + refreshLatestAck: boolean, + /** Logger to use for correlated summary events */ + summaryLogger: ITelemetryLogger, +} + export interface ISummarizerInternalsProvider { /** Encapsulates the work to walk the internals of the running container to generate a summary */ - generateSummary( - full: boolean, - safe: boolean, - summaryLogger: ITelemetryLogger, - ): Promise; + generateSummary(options: IGenerateSummaryOptions): Promise; /** Callback whenever a new SummaryAck is received, to update internal tracking state */ refreshLatestSummaryAck( @@ -563,7 +568,11 @@ export class RunningSummarizer implements IDisposable { // Wait for generate/send summary let summaryData: GenerateSummaryData | undefined; try { - summaryData = await this.internalsProvider.generateSummary(this.immediateSummary, safe, this.logger); + summaryData = await this.internalsProvider.generateSummary({ + fullTree: this.immediateSummary || safe, + refreshLatestAck: safe, + summaryLogger: this.logger, + }); } catch (error) { summarizingEvent.cancel({ category: "error" }, error); return; @@ -839,11 +848,11 @@ export class Summarizer extends EventEmitter implements ISummarizer { } /** Implementation of SummarizerInternalsProvider.generateSummary */ - public async generateSummary( - full: boolean, - safe: boolean, + public async generateSummary(options: { + fullTree: boolean, + refreshLatestAck: boolean, summaryLogger: ITelemetryLogger, - ): Promise { + }): Promise { if (this.onBehalfOfClientId !== this.runtime.summarizerClientId && this.runtime.clientId !== this.runtime.summarizerClientId) { // We are no longer the summarizer; a different client is, so we should stop ourself @@ -851,7 +860,7 @@ export class Summarizer extends EventEmitter implements ISummarizer { return undefined; } - return this.internalsProvider.generateSummary(full, safe, summaryLogger); + return this.internalsProvider.generateSummary(options); } private async handleSummaryAcks() { From 420f6a07508969b0c626c6e609ced01bb68927c8 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 25 Feb 2021 18:07:47 -0500 Subject: [PATCH 03/35] Add simple summaries end-to-end test --- .../container-runtime/src/containerRuntime.ts | 6 +- .../src/test/summaries.spec.ts | 118 ++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 packages/test/end-to-end-tests/src/test/summaries.spec.ts diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 242c905568cc..29ce0896aa16 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -1458,11 +1458,11 @@ export class ContainerRuntime extends TypedEventEmitter ); } - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const parent = this.latestSummaryAck.ackHandle!; + const parent = this.latestSummaryAck.ackHandle; const summaryMessage: ISummaryContent = { handle, - head: parent, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + head: parent!, message, parents: parent ? [parent] : [], }; diff --git a/packages/test/end-to-end-tests/src/test/summaries.spec.ts b/packages/test/end-to-end-tests/src/test/summaries.spec.ts new file mode 100644 index 000000000000..9644d538ae68 --- /dev/null +++ b/packages/test/end-to-end-tests/src/test/summaries.spec.ts @@ -0,0 +1,118 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +import { ContainerRuntimeFactoryWithDefaultDataStore, DataObject, DataObjectFactory } from "@fluidframework/aqueduct"; +import { assert, TelemetryNullLogger } from "@fluidframework/common-utils"; +import { IContainer } from "@fluidframework/container-definitions"; +import { ContainerRuntime, IContainerRuntimeOptions } from "@fluidframework/container-runtime"; +import { IFluidCodeDetails } from "@fluidframework/core-interfaces"; +import { LocalResolver } from "@fluidframework/local-driver"; +import { SharedDirectory, SharedMap } from "@fluidframework/map"; +import { SharedMatrix } from "@fluidframework/matrix"; +import { SummaryType } from "@fluidframework/protocol-definitions"; +import { requestFluidObject } from "@fluidframework/runtime-utils"; +import { SharedObjectSequence } from "@fluidframework/sequence"; +import { LocalDeltaConnectionServer } from "@fluidframework/server-local-server"; +import { createAndAttachContainer, createLocalLoader, OpProcessingController } from "@fluidframework/test-utils"; + +const defaultDataStoreId = "default"; + +class TestDataObject extends DataObject { + public static readonly dataObjectName = "TestDataObject"; + public readonly getRoot = () => this.root; + public readonly getRuntime = () => this.runtime; + public readonly getContext = () => this.context; +} + +async function createContainer(): Promise<{ + container: IContainer; + opProcessingController: OpProcessingController; +}> { + const documentId = "summarizerTest"; + + const codeDetails: IFluidCodeDetails = { + package: "summarizerTestPackage", + }; + + const factory = new DataObjectFactory(TestDataObject.dataObjectName, TestDataObject, [ + SharedMap.getFactory(), + SharedDirectory.getFactory(), + SharedMatrix.getFactory(), + SharedObjectSequence.getFactory(), + ], []); + + const runtimeOptions: IContainerRuntimeOptions = { + generateSummaries: false, + }; + + const runtimeFactory = new ContainerRuntimeFactoryWithDefaultDataStore( + factory, + [ + [defaultDataStoreId, Promise.resolve(factory)], + [TestDataObject.dataObjectName, Promise.resolve(factory)], + ], + undefined, + undefined, + runtimeOptions, + ); + + const deltaConnectionServer = LocalDeltaConnectionServer.create(); + + const urlResolver = new LocalResolver(); + + const loader = createLocalLoader([[codeDetails, runtimeFactory]], deltaConnectionServer, urlResolver); + const container = await createAndAttachContainer( + codeDetails, + loader, + urlResolver.createCreateNewRequest(documentId)); + + const opProcessingController = new OpProcessingController(); + opProcessingController.addDeltaManagers(container.deltaManager); + + return { container, opProcessingController }; +} + +describe("Summaries", () => { + it("Should generate summary tree", async () => { + const { container, opProcessingController } = await createContainer(); + const defaultDataStore = await requestFluidObject(container, defaultDataStoreId); + const containerRuntime = defaultDataStore.getContext().containerRuntime as ContainerRuntime; + + await opProcessingController.process(); + + const { gcData, stats, summary } = await containerRuntime.summarize({ + runGc: false, + fullTree: false, + trackState: false, + summaryLogger: new TelemetryNullLogger(), + }); + + // Validate stats + assert(stats.handleNodeCount === 0, "Expecting no handles for first summary."); + // .component and .attributes blobs + assert(stats.blobNodeCount >= 2, `Stats expected at least 2 blob nodes, but had ${stats.blobNodeCount}.`); + // root node, default data store, and default root dds + assert(stats.treeNodeCount >= 3, `Stats expected at least 3 tree nodes, but had ${stats.treeNodeCount}.`); + + // Validate summary + assert(!summary.unreferenced, "Root summary should be referenced."); + const defaultDataStoreNode = summary.tree[defaultDataStoreId]; + assert(defaultDataStoreNode?.type === SummaryType.Tree, "Expected default data store tree in summary."); + assert(!defaultDataStoreNode.unreferenced, "Default data store should be referenced."); + assert(defaultDataStoreNode.tree[".component"].type === SummaryType.Blob, + "Expected .component blob in default data store summary tree."); + const defaultDdsNode = defaultDataStoreNode.tree.root; + assert(defaultDdsNode?.type === SummaryType.Tree, "Expected default root DDS in summary."); + assert(!defaultDdsNode.unreferenced, "Default root DDS should be referenced."); + assert(defaultDdsNode.tree[".attributes"].type === SummaryType.Blob, + "Expected .attributes blob in default root DDS summary tree."); + + // Validate GC nodes + const gcNodeIds = Object.keys(gcData.gcNodes); + assert(gcNodeIds.includes("/"), "Expected root gc node."); + assert(gcNodeIds.includes("/default"), "Expected default data store gc node."); + assert(gcNodeIds.includes("/default/root"), "Expected default root DDS gc node."); + }); +}); From d0f932ac83d98c07d15d76c139cf47100be38b9c Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 26 Feb 2021 03:38:12 -0500 Subject: [PATCH 04/35] Increment summary format version and put data stores/dds under .channels subtrees --- .../container-runtime/src/containerRuntime.ts | 27 ++++++++-- .../container-runtime/src/dataStoreContext.ts | 12 +++-- .../container-runtime/src/dataStores.ts | 6 +-- .../src/{snapshot.ts => summaryFormat.ts} | 53 ++++++++++++------- .../src/test/dataStoreContext.spec.ts | 2 +- .../src/test/dataStores.spec.ts | 2 +- 6 files changed, 68 insertions(+), 34 deletions(-) rename packages/runtime/container-runtime/src/{snapshot.ts => summaryFormat.ts} (50%) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index f34e2ef953ff..3dae6472ae5d 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -56,6 +56,7 @@ import { CreateContainerError } from "@fluidframework/container-utils"; import { runGarbageCollection } from "@fluidframework/garbage-collector"; import { BlobTreeEntry, + TreeTreeEntry, } from "@fluidframework/protocol-base"; import { IClientDetails, @@ -94,6 +95,7 @@ import { IChannelSummarizeResult, CreateChildSummarizerNodeParam, SummarizeInternalFn, + channelsTreeName, } from "@fluidframework/runtime-definitions"; import { addBlobToSummary, @@ -124,7 +126,8 @@ import { chunksBlobName, IContainerRuntimeMetadata, metadataBlobName, -} from "./snapshot"; + wrapSummaryInChannelsTree, +} from "./summaryFormat"; export enum ContainerMessageType { // An op to be delivered to store @@ -943,7 +946,15 @@ export class ContainerRuntime extends TypedEventEmitter * @deprecated - Use summarize to get summary of the container runtime. */ public async snapshot(): Promise { - const root: ITree = { entries: await this.dataStores.snapshot() }; + const root: ITree = { entries: [] }; + + root.entries.push(new TreeTreeEntry( + channelsTreeName, + { entries: await this.dataStores.snapshot() }, + )); + + const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; + root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); if (this.chunkMap.size > 0) { root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap]))); @@ -953,6 +964,8 @@ export class ContainerRuntime extends TypedEventEmitter } private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) { + const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; + addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); if (this.chunkMap.size > 0) { const content = JSON.stringify([...this.chunkMap]); addBlobToSummary(summaryTree, chunksBlobName, content); @@ -1346,6 +1359,8 @@ export class ContainerRuntime extends TypedEventEmitter private async summarizeInternal(fullTree: boolean, trackState: boolean): Promise { const summarizeResult = await this.dataStores.summarize(fullTree, trackState); + // Wrap data store summaries in .channels subtree. + wrapSummaryInChannelsTree(summarizeResult); this.addContainerBlobsToSummary(summarizeResult); return { ...summarizeResult, @@ -1364,9 +1379,11 @@ export class ContainerRuntime extends TypedEventEmitter } public createSummary(): ISummaryTree { - const summaryTree = this.dataStores.createSummary(); - this.addContainerBlobsToSummary(summaryTree); - return summaryTree.summary; + const summarizeResult = this.dataStores.createSummary(); + // Wrap data store summaries in .channels subtree. + wrapSummaryInChannelsTree(summarizeResult); + this.addContainerBlobsToSummary(summarizeResult); + return summarizeResult.summary; } public async getAbsoluteUrl(relativeUrl: string): Promise { diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index b55d4e3eb5c5..c6df271af588 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -64,14 +64,15 @@ import { addBlobToSummary, convertSummaryTreeToITree } from "@fluidframework/run import { ContainerRuntime } from "./containerRuntime"; import { dataStoreAttributesBlobName, - DataStoreSnapshotFormatVersion, -} from "./snapshot"; + DataStoreSummaryFormatVersion, + wrapSummaryInChannelsTree, +} from "./summaryFormat"; function createAttributes(pkg: readonly string[], isRootDataStore: boolean): IFluidDataStoreAttributes { const stringifiedPkg = JSON.stringify(pkg); return { pkg: stringifiedPkg, - snapshotFormatVersion: "0.1", + snapshotFormatVersion: 2, isRootDataStore, }; } @@ -87,7 +88,7 @@ export function createAttributesBlob(pkg: readonly string[], isRootDataStore: bo */ export interface IFluidDataStoreAttributes { pkg: string; - readonly snapshotFormatVersion: DataStoreSnapshotFormatVersion; + readonly snapshotFormatVersion: DataStoreSummaryFormatVersion; /** * This tells whether a data store is root. Root data stores are never collected. * Non-root data stores may be collected if they are not used. If this is not present, default it to @@ -382,6 +383,9 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter { const dataStoreId = "Test1"; diff --git a/packages/runtime/container-runtime/src/test/dataStores.spec.ts b/packages/runtime/container-runtime/src/test/dataStores.spec.ts index 0c932e7ae6ba..cc09dbd88da4 100644 --- a/packages/runtime/container-runtime/src/test/dataStores.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStores.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "assert"; import { ISnapshotTree } from "@fluidframework/protocol-definitions"; import { channelsTreeName } from "@fluidframework/runtime-definitions"; import { getSnapshotForDataStores } from "../dataStores"; -import { nonDataStorePaths } from "../snapshot"; +import { nonDataStorePaths } from "../summaryFormat"; describe("Runtime", () => { describe("Container Runtime", () => { From 5dfb0cff4705de394a572665146e7445308119e6 Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 26 Feb 2021 04:45:11 -0500 Subject: [PATCH 05/35] Fix some tests --- .../container-runtime/src/test/dataStoreContext.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index fafa9348ee85..a6224b0097d7 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -110,7 +110,7 @@ describe("Data Store Context Tests", () => { const contents = JSON.parse((attributesEntry.value as IBlob).contents) as IFluidDataStoreAttributes; const dataStoreAttributes: IFluidDataStoreAttributes = { pkg: JSON.stringify(["TestDataStore1"]), - snapshotFormatVersion: "0.1", + snapshotFormatVersion: 2, isRootDataStore: true, }; @@ -180,7 +180,7 @@ describe("Data Store Context Tests", () => { const contents = JSON.parse((attributesEntry.value as IBlob).contents) as IFluidDataStoreAttributes; const dataStoreAttributes: IFluidDataStoreAttributes = { pkg: JSON.stringify(["TestComp", "SubComp"]), - snapshotFormatVersion: "0.1", + snapshotFormatVersion: 2, isRootDataStore: false, }; @@ -337,7 +337,7 @@ describe("Data Store Context Tests", () => { it("can correctly initialize and generate attributes", async () => { dataStoreAttributes = { pkg: JSON.stringify(["TestDataStore1"]), - snapshotFormatVersion: "0.1", + snapshotFormatVersion: 2, isRootDataStore: true, }; const buffer = IsoBuffer.from(JSON.stringify(dataStoreAttributes), "utf-8"); @@ -416,7 +416,7 @@ describe("Data Store Context Tests", () => { "Remote DataStore package does not match."); assert.strictEqual( contents.snapshotFormatVersion, - "0.1", + 2, "Remote DataStore snapshot version does not match."); // Remote context without the isRootDataStore flag in the snapshot should default it to true. assert.strictEqual(contents.isRootDataStore, true, "Remote DataStore root state does not match."); From 205a66bff0ec0efbaf0ff3fa0733866f01a7846d Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 26 Feb 2021 17:51:17 -0500 Subject: [PATCH 06/35] Update basic summary test for subtrees --- .../src/test/summaries.spec.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/test/end-to-end-tests/src/test/summaries.spec.ts b/packages/test/end-to-end-tests/src/test/summaries.spec.ts index 9644d538ae68..ab8e7098ce6f 100644 --- a/packages/test/end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/end-to-end-tests/src/test/summaries.spec.ts @@ -12,6 +12,7 @@ import { LocalResolver } from "@fluidframework/local-driver"; import { SharedDirectory, SharedMap } from "@fluidframework/map"; import { SharedMatrix } from "@fluidframework/matrix"; import { SummaryType } from "@fluidframework/protocol-definitions"; +import { channelsTreeName } from "@fluidframework/runtime-definitions"; import { requestFluidObject } from "@fluidframework/runtime-utils"; import { SharedObjectSequence } from "@fluidframework/sequence"; import { LocalDeltaConnectionServer } from "@fluidframework/server-local-server"; @@ -91,22 +92,31 @@ describe("Summaries", () => { // Validate stats assert(stats.handleNodeCount === 0, "Expecting no handles for first summary."); - // .component and .attributes blobs - assert(stats.blobNodeCount >= 2, `Stats expected at least 2 blob nodes, but had ${stats.blobNodeCount}.`); - // root node, default data store, and default root dds - assert(stats.treeNodeCount >= 3, `Stats expected at least 3 tree nodes, but had ${stats.treeNodeCount}.`); + // .metadata, .component, and .attributes blobs + assert(stats.blobNodeCount >= 3, `Stats expected at least 2 blob nodes, but had ${stats.blobNodeCount}.`); + // root node, data store .channels, default data store, dds .channels, and default root dds + assert(stats.treeNodeCount >= 5, `Stats expected at least 3 tree nodes, but had ${stats.treeNodeCount}.`); // Validate summary assert(!summary.unreferenced, "Root summary should be referenced."); - const defaultDataStoreNode = summary.tree[defaultDataStoreId]; + + assert(summary.tree[".metadata"]?.type === SummaryType.Blob, "Expected .metadata blob in summary root."); + + const channelsTree = summary.tree[channelsTreeName]; + assert(channelsTree?.type === SummaryType.Tree, "Expected .channels tree in summary root."); + + const defaultDataStoreNode = channelsTree.tree[defaultDataStoreId]; assert(defaultDataStoreNode?.type === SummaryType.Tree, "Expected default data store tree in summary."); assert(!defaultDataStoreNode.unreferenced, "Default data store should be referenced."); - assert(defaultDataStoreNode.tree[".component"].type === SummaryType.Blob, + assert(defaultDataStoreNode.tree[".component"]?.type === SummaryType.Blob, "Expected .component blob in default data store summary tree."); - const defaultDdsNode = defaultDataStoreNode.tree.root; + const dataStoreChannelsTree = defaultDataStoreNode.tree[channelsTreeName]; + assert(dataStoreChannelsTree?.type === SummaryType.Tree, "Expected .channels tree in default data store."); + + const defaultDdsNode = dataStoreChannelsTree.tree.root; assert(defaultDdsNode?.type === SummaryType.Tree, "Expected default root DDS in summary."); assert(!defaultDdsNode.unreferenced, "Default root DDS should be referenced."); - assert(defaultDdsNode.tree[".attributes"].type === SummaryType.Blob, + assert(defaultDdsNode.tree[".attributes"]?.type === SummaryType.Blob, "Expected .attributes blob in default root DDS summary tree."); // Validate GC nodes From 4390009fbcaa9d9b627d813526c2c068922ea66e Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 2 Mar 2021 14:21:40 -0500 Subject: [PATCH 07/35] Write in new format in all places --- .../runtime/container-runtime/src/containerRuntime.ts | 5 +++++ .../runtime/container-runtime/src/dataStoreContext.ts | 9 ++++++++- packages/runtime/runtime-definitions/src/summary.ts | 8 ++++++++ .../src/summarizerNode/summarizerNode.ts | 11 +++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index e4a0313aca8d..fb77dcbee0e9 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -723,6 +723,10 @@ export class ContainerRuntime extends TypedEventEmitter }, ); + if (this.context.baseSnapshot) { + this.summarizerNode.loadBaseSummaryWithoutDifferential(this.context.baseSnapshot); + } + this.dataStores = new DataStores( getSnapshotForDataStores(context.baseSnapshot, metadata.snapshotFormatVersion), this, @@ -1412,6 +1416,7 @@ export class ContainerRuntime extends TypedEventEmitter return { ...summarizeResult, id: "", + pathPartsForChildren: [channelsTreeName], }; } diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index c6df271af588..8b837ebe3588 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -403,7 +403,11 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter Promise; @@ -102,6 +104,12 @@ export interface ISummarizerNode { * @param fullTree - true to skip optimizations and always generate the full tree */ summarize(fullTree: boolean): Promise; + /** + * Checks if there are any additional path parts for children that need to + * be loaded from the base summary. + * @param snapshot - the base summary to parse + */ + loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree): void; /** * Checks if the base snapshot was created as a failure summary. If it has * the base summary handle + outstanding ops blob, then this will return the diff --git a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts index e5cbb78003c1..ef47ec89af00 100644 --- a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts +++ b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts @@ -106,6 +106,9 @@ export class SummarizerNode implements IRootSummarizerNode { try { const result = await this.summarizeInternalFn(fullTree); this.wipLocalPaths = { localPath: EscapedPath.create(result.id) }; + if (result.pathPartsForChildren !== undefined) { + this.wipLocalPaths.additionalPath = EscapedPath.createAndConcat(result.pathPartsForChildren); + } return { summary: result.summary, stats: result.stats }; } catch (error) { if (this.throwOnError || this.trackingSequenceNumber < this._changeSequenceNumber) { @@ -358,6 +361,14 @@ export class SummarizerNode implements IRootSummarizerNode { } } + public loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree) { + const { childrenPathPart } = parseSummaryForSubtrees(snapshot); + if (childrenPathPart !== undefined) { + assert(!!this.latestSummary, "Should have latest summary defined during loadBaseSummary"); + this.latestSummary.additionalPath = EscapedPath.create(childrenPathPart); + } + } + public async loadBaseSummary( snapshot: ISnapshotTree, readAndParseBlob: ReadAndParseBlob, From 791c9ab9f0e7e7104133f19c368068c7a71d8c4a Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 2 Mar 2021 20:17:18 -0500 Subject: [PATCH 08/35] Add runtime option to disable isolated channels --- .../container-runtime/src/containerRuntime.ts | 42 +++++++++++------ .../container-runtime/src/dataStoreContext.ts | 45 ++++++++++++++----- .../container-runtime/src/dataStores.ts | 13 ++++-- .../src/test/dataStoreContext.spec.ts | 28 +++++++++--- .../src/test/dataStoreCreation.spec.ts | 27 +++++++---- 5 files changed, 112 insertions(+), 43 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index fb77dcbee0e9..2e6a2fd66b01 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -230,6 +230,10 @@ export interface IContainerRuntimeOptions { // Override summary configurations summaryConfigOverrides?: Partial; + + // Flag that disables putting channels in isolated subtrees for each data store + // and the root node when generating a summary if set to true. + disableIsolatedChannels?: boolean; } interface IRuntimeMessageMetadata { @@ -743,6 +747,7 @@ export class ContainerRuntime extends TypedEventEmitter getGCDataFn, getInitialGCSummaryDetailsFn, ), + !!this.runtimeOptions.disableIsolatedChannels, this._logger); this.blobManager = new BlobManager( @@ -952,13 +957,15 @@ export class ContainerRuntime extends TypedEventEmitter public async snapshot(): Promise { const root: ITree = { entries: [] }; - root.entries.push(new TreeTreeEntry( - channelsTreeName, - { entries: await this.dataStores.snapshot() }, - )); + if (!this.runtimeOptions.disableIsolatedChannels) { + root.entries.push(new TreeTreeEntry( + channelsTreeName, + { entries: await this.dataStores.snapshot() }, + )); - const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; - root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); + const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; + root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); + } if (this.chunkMap.size > 0) { root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap]))); @@ -968,8 +975,10 @@ export class ContainerRuntime extends TypedEventEmitter } private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) { - const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; - addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); + if (!this.runtimeOptions.disableIsolatedChannels) { + const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; + addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); + } if (this.chunkMap.size > 0) { const content = JSON.stringify([...this.chunkMap]); addBlobToSummary(summaryTree, chunksBlobName, content); @@ -1361,8 +1370,10 @@ export class ContainerRuntime extends TypedEventEmitter public createSummary(): ISummaryTree { const summarizeResult = this.dataStores.createSummary(); - // Wrap data store summaries in .channels subtree. - wrapSummaryInChannelsTree(summarizeResult); + if (!this.runtimeOptions.disableIsolatedChannels) { + // Wrap data store summaries in .channels subtree. + wrapSummaryInChannelsTree(summarizeResult); + } this.addContainerBlobsToSummary(summarizeResult); return summarizeResult.summary; } @@ -1410,13 +1421,18 @@ export class ContainerRuntime extends TypedEventEmitter private async summarizeInternal(fullTree: boolean, trackState: boolean): Promise { const summarizeResult = await this.dataStores.summarize(fullTree, trackState); - // Wrap data store summaries in .channels subtree. - wrapSummaryInChannelsTree(summarizeResult); + let pathPartsForChildren: string[] | undefined; + + if (!this.runtimeOptions.disableIsolatedChannels) { + // Wrap data store summaries in .channels subtree. + wrapSummaryInChannelsTree(summarizeResult); + pathPartsForChildren = [channelsTreeName]; + } this.addContainerBlobsToSummary(summarizeResult); return { ...summarizeResult, id: "", - pathPartsForChildren: [channelsTreeName], + pathPartsForChildren, }; } diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 8b837ebe3588..fb70ff73134b 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -68,16 +68,24 @@ import { wrapSummaryInChannelsTree, } from "./summaryFormat"; -function createAttributes(pkg: readonly string[], isRootDataStore: boolean): IFluidDataStoreAttributes { +function createAttributes( + pkg: readonly string[], + isRootDataStore: boolean, + disableIsolatedChannels: boolean, +): IFluidDataStoreAttributes { const stringifiedPkg = JSON.stringify(pkg); return { pkg: stringifiedPkg, - snapshotFormatVersion: 2, + snapshotFormatVersion: disableIsolatedChannels ? "0.1" : 2, isRootDataStore, }; } -export function createAttributesBlob(pkg: readonly string[], isRootDataStore: boolean): ITreeEntry { - const attributes = createAttributes(pkg, isRootDataStore); +export function createAttributesBlob( + pkg: readonly string[], + isRootDataStore: boolean, + disableIsolatedChannels: boolean, +): ITreeEntry { + const attributes = createAttributes(pkg, isRootDataStore, disableIsolatedChannels); return new BlobTreeEntry(dataStoreAttributesBlobName, JSON.stringify(attributes)); } @@ -200,6 +208,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter void, + protected readonly disableIsolatedChannels: boolean, protected pkg?: readonly string[], ) { super(); @@ -382,13 +391,17 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter { throw new Error("Already attached"); }, + disableIsolatedChannels, pkg, ); } @@ -776,6 +791,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { bindChannel: (channel: IFluidDataStoreChannel) => void, private readonly snapshotTree: ISnapshotTree | undefined, protected readonly isRootDataStore: boolean, + disableIsolatedChannels: boolean, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -791,6 +807,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { snapshotTree ? BindState.Bound : BindState.NotBound, true, bindChannel, + disableIsolatedChannels, pkg); this.attachListeners(); } @@ -813,11 +830,13 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { const summarizeResult = this.channel.getAttachSummary(); - // Wrap dds summaries in .channels subtree. - wrapSummaryInChannelsTree(summarizeResult); + if (!this.disableIsolatedChannels) { + // Wrap dds summaries in .channels subtree. + wrapSummaryInChannelsTree(summarizeResult); + } // Add data store's attributes to the summary. - const attributes: IFluidDataStoreAttributes = createAttributes(this.pkg, this.isRootDataStore); + const attributes = createAttributes(this.pkg, this.isRootDataStore, this.disableIsolatedChannels); addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes)); // Add GC details to the summary. @@ -872,6 +891,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { bindChannel: (channel: IFluidDataStoreChannel) => void, snapshotTree: ISnapshotTree | undefined, isRootDataStore: boolean, + disableIsolatedChannels: boolean, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -887,6 +907,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { bindChannel, snapshotTree, isRootDataStore, + disableIsolatedChannels, createProps); } } @@ -911,6 +932,7 @@ export class LocalDetachedFluidDataStoreContext bindChannel: (channel: IFluidDataStoreChannel) => void, snapshotTree: ISnapshotTree | undefined, isRootDataStore: boolean, + disableIsolatedChannels: boolean, ) { super( id, @@ -922,6 +944,7 @@ export class LocalDetachedFluidDataStoreContext bindChannel, snapshotTree, isRootDataStore, + disableIsolatedChannels, ); this.detachedRuntimeCreation = true; } diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index 7c98d276ad89..61b0c9ca1ce3 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -76,6 +76,7 @@ export class DataStores implements IDisposable { private readonly submitAttachFn: (attachContent: any) => void, private readonly getCreateChildSummarizerNodeFn: (id: string, createParam: CreateChildSummarizerNodeParam) => CreateChildSummarizerNodeFn, + private readonly disableIsolatedChannels: boolean, baseLogger: ITelemetryBaseLogger, private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger), ) { @@ -100,7 +101,8 @@ export class DataStores implements IDisposable { this.runtime, this.runtime.storage, this.runtime.scope, - this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary })); + this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary }), + this.disableIsolatedChannels); } else { let pkgFromSnapshot: string[]; if (typeof value !== "object") { @@ -134,7 +136,9 @@ export class DataStores implements IDisposable { // If there is no isRootDataStore in the attributes blob, set it to true. This ensures that data // stores in older documents are not garbage collected incorrectly. This may lead to additional // roots in the document but they won't break. - attributes.isRootDataStore ?? true); + attributes.isRootDataStore ?? true, + this.disableIsolatedChannels, + ); } this.contexts.addBoundOrRemoted(dataStoreContext); } @@ -185,9 +189,10 @@ export class DataStores implements IDisposable { type: CreateSummarizerNodeSource.FromAttach, sequenceNumber: message.sequenceNumber, snapshot: attachMessage.snapshot ?? { - entries: [createAttributesBlob(pkg, true /* isRootDataStore */)], + entries: [createAttributesBlob(pkg, true /* isRootDataStore */, this.disableIsolatedChannels)], }, }), + this.disableIsolatedChannels, pkg); // Resolve pending gets and store off any new ones @@ -233,6 +238,7 @@ export class DataStores implements IDisposable { (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr), undefined, isRoot, + this.disableIsolatedChannels, ); this.contexts.addUnbound(context); return context; @@ -249,6 +255,7 @@ export class DataStores implements IDisposable { (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr), undefined, isRoot, + this.disableIsolatedChannels, props, ); this.contexts.addUnbound(context); diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index a6224b0097d7..0cc03e4d61d2 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -99,7 +99,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */); + true /* isRootDataStore */, + false /* disableIsolatedChannels */); await localDataStoreContext.realize(); const attachMessage = localDataStoreContext.generateAttachMessage(); @@ -137,7 +138,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); await localDataStoreContext.realize() .catch((error) => { @@ -169,7 +171,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); await localDataStoreContext.realize(); @@ -206,7 +209,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */); + true /* isRootDataStore */, + false /* disableIsolatedChannels */); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, true, "The data store should be root."); @@ -222,7 +226,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, false, "The data store should not be root."); @@ -240,7 +245,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */); + true /* isRootDataStore */, + false /* disableIsolatedChannels */); const gcData = await localDataStoreContext.getGCData(); assert.deepStrictEqual(gcData, emptyGCData, "GC data from getGCData should be empty."); @@ -265,7 +271,8 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); // Get the summarizer node for this data store which tracks its referenced state. const dataStoreSummarizerNode = summarizerNode.getChild(dataStoreId); @@ -355,6 +362,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); const isRootNode = await remotedDataStoreContext.isRoot(); @@ -399,6 +407,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); const isRootNode = await remotedDataStoreContext.isRoot(); @@ -463,6 +472,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -511,6 +521,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -566,6 +577,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -603,6 +615,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); // Update used routes before calling summarize. This is a requirement for GC. @@ -644,6 +657,7 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, + false /* disableIsolatedChannels */, ); // Get the summarizer node for this data store which tracks its referenced state. diff --git a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts index ff967dfd7195..a43df5041abd 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts @@ -121,7 +121,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await context.realize(); @@ -145,7 +146,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await context.realize(); @@ -169,7 +171,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextA.realize(); @@ -193,7 +196,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextB.realize(); @@ -217,7 +221,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreBId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextB.realize(); @@ -238,7 +243,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreCId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextC.realize(); @@ -262,7 +268,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextFake.realize(); @@ -286,7 +293,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextFake.realize(); @@ -310,7 +318,8 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */); + false /* isRootDataStore */, + false /* disableIsolatedChannels */); try { await contextC.realize(); From a3890970e31e6da8dd82c34529b83beeea62d527 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 4 Mar 2021 04:20:01 -0500 Subject: [PATCH 09/35] Fix handling initial summary from attach op --- .../src/summarizerNode/summarizerNode.ts | 20 +++++++++---- .../src/summarizerNode/summarizerNodeUtils.ts | 28 +++++++++++++++++-- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts index ef47ec89af00..bca46252cf62 100644 --- a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts +++ b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts @@ -16,6 +16,7 @@ import { ISequencedDocumentMessage, SummaryType, ISnapshotTree, + SummaryObject, } from "@fluidframework/protocol-definitions"; import { ITelemetryLogger } from "@fluidframework/common-definitions"; import { assert, unreachableCase } from "@fluidframework/common-utils"; @@ -29,6 +30,7 @@ import { IInitialSummary, ISummarizerNodeRootContract, parseSummaryForSubtrees, + parseSummaryTreeForSubtrees, ReadAndParseBlob, seqFromTree, SummaryNode, @@ -363,8 +365,7 @@ export class SummarizerNode implements IRootSummarizerNode { public loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree) { const { childrenPathPart } = parseSummaryForSubtrees(snapshot); - if (childrenPathPart !== undefined) { - assert(!!this.latestSummary, "Should have latest summary defined during loadBaseSummary"); + if (childrenPathPart !== undefined && this.latestSummary !== undefined) { this.latestSummary.additionalPath = EscapedPath.create(childrenPathPart); } } @@ -381,8 +382,7 @@ export class SummarizerNode implements IRootSummarizerNode { decodedSummary.pathParts.push(childrenPathPart); } - if (decodedSummary.pathParts.length > 0) { - assert(!!this.latestSummary, "Should have latest summary defined during loadBaseSummary"); + if (decodedSummary.pathParts.length > 0 && this.latestSummary !== undefined) { this.latestSummary.additionalPath = EscapedPath.createAndConcat(decodedSummary.pathParts); } @@ -536,9 +536,17 @@ export class SummarizerNode implements IRootSummarizerNode { case CreateSummarizerNodeSource.Local: { const parentInitialSummary = this.initialSummary; if (parentInitialSummary !== undefined) { - const childSummary = parentInitialSummary.summary?.summary.tree[id]; + let childSummary: SummaryObject | undefined; + if (parentInitialSummary.summary !== undefined) { + const { childrenTree } = parseSummaryTreeForSubtrees(parentInitialSummary.summary.summary); + assert( + childrenTree.type === SummaryType.Tree, + "Parent summary object is not a tree", + ); + childSummary = childrenTree.tree[id]; + } if (createParam.type === CreateSummarizerNodeSource.FromSummary) { - // Locally created would not have subtree. + // Locally created would not have differential subtree. assert(!!childSummary, "Missing child summary tree"); } let childSummaryWithStats: ISummaryTreeWithStats | undefined; diff --git a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNodeUtils.ts b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNodeUtils.ts index a66dd688e0df..b88d289a16f5 100644 --- a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNodeUtils.ts +++ b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNodeUtils.ts @@ -10,6 +10,8 @@ import { IDocumentAttributes, ISequencedDocumentMessage, SummaryType, + ISummaryTree, + SummaryObject, } from "@fluidframework/protocol-definitions"; import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions"; import { SummaryTreeBuilder } from "../summaryUtils"; @@ -289,9 +291,9 @@ export interface ICreateChildDetails { changeSequenceNumber: number; } -export interface ISubtreeInfo { +export interface ISubtreeInfo { /** Tree to use to find children subtrees */ - childrenTree: ISnapshotTree, + childrenTree: T, /** Additional path part where children are isolated */ childrenPathPart: string | undefined, } @@ -301,7 +303,7 @@ export interface ISubtreeInfo { * would be located if exists. * @param baseSummary - summary to check */ -export function parseSummaryForSubtrees(baseSummary: ISnapshotTree): ISubtreeInfo { +export function parseSummaryForSubtrees(baseSummary: ISnapshotTree): ISubtreeInfo { // New versions of snapshots have child nodes isolated in .channels subtree const channelsSubtree = baseSummary.trees[channelsTreeName]; if (channelsSubtree !== undefined) { @@ -315,3 +317,23 @@ export function parseSummaryForSubtrees(baseSummary: ISnapshotTree): ISubtreeInf childrenPathPart: undefined, }; } + +/** + * Checks if the summary contains .channels subtree where the children subtrees + * would be located if exists. + * @param baseSummary - summary to check + */ +export function parseSummaryTreeForSubtrees(summary: ISummaryTree): ISubtreeInfo { + // New versions of snapshots have child nodes isolated in .channels subtree + const channelsSubtree = summary.tree[channelsTreeName]; + if (channelsSubtree !== undefined) { + return { + childrenTree: channelsSubtree, + childrenPathPart: channelsTreeName, + }; + } + return { + childrenTree: summary, + childrenPathPart: undefined, + }; +} From 8969b1175b59833f45c2b30644572a692d702b89 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 4 Mar 2021 07:05:22 -0500 Subject: [PATCH 10/35] Strip out new format version features from snapshots in comparison --- packages/tools/replay-tool/src/helpers.ts | 14 ++++++++++--- .../tool-utils/src/snapshotNormalizer.ts | 20 +++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/tools/replay-tool/src/helpers.ts b/packages/tools/replay-tool/src/helpers.ts index 02d5d971cce9..460f33dacdb2 100644 --- a/packages/tools/replay-tool/src/helpers.ts +++ b/packages/tools/replay-tool/src/helpers.ts @@ -63,14 +63,22 @@ export function compareWithReferenceSnapshot( * Before replace - "{\"type\":\"https://graph.microsoft.com/types/map\",\"packageVersion\":\"0.28.0-214\"}" * After replace - "{\"type\":\"https://graph.microsoft.com/types/map\",\"packageVersion\":\"X\"}" */ - const packageVersionRegex = /\\"packageversion\\":\\".+\\"/gi; + const packageVersionRegex = /\\"packageversion\\":\\"[^"]+\\"/gi; const packageVersionPlaceholder = "\\\"packageVersion\\\":\\\"X\\\""; + // The snapshotFormatVersion could vary. Replace all with -1 like packageVersion. + const snapshotFormatVersionRegex = /\\"snapshotformatversion\\":(\\"[^"]+\\"|\d+)/gi; + const snapshotFormatVersionPlaceholder = "\\\"snapshotFormatVersion\\\":-1"; + const normalizedSnapshot = JSON.parse( - JSON.stringify(snapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder), + JSON.stringify(snapshot, undefined, 2) + .replace(packageVersionRegex, packageVersionPlaceholder) + .replace(snapshotFormatVersionRegex, snapshotFormatVersionPlaceholder), ); const normalizedReferenceSnapshot = JSON.parse( - JSON.stringify(referenceSnapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder), + JSON.stringify(referenceSnapshot, undefined, 2) + .replace(packageVersionRegex, packageVersionPlaceholder) + .replace(snapshotFormatVersionRegex, snapshotFormatVersionPlaceholder), ); // Put the assert in a try catch block, so that we can report errors, if any. diff --git a/packages/utils/tool-utils/src/snapshotNormalizer.ts b/packages/utils/tool-utils/src/snapshotNormalizer.ts index 3555228b0e83..3ef8dcbcb41f 100644 --- a/packages/utils/tool-utils/src/snapshotNormalizer.ts +++ b/packages/utils/tool-utils/src/snapshotNormalizer.ts @@ -92,12 +92,16 @@ function getSortedBlobContent(content: string): string { * Helper function that normalizes the given snapshot tree. It sorts objects and arrays in the snapshot. It also * normalizes certain blob contents for which the order of content does not matter. For example, garbage collection * blobs contains objects / arrays whose element order do not matter. + * We also promote any .channels subtrees, and exclude the root .metadata blob for backwards compatibility. + * In a future version, once we can reasonably expect most/all of our test criteria to have the new format with + * .channels subtrees and .metadata blob, we should invert this logic to expect .channels and .metadata, + * and add specifically for the tests (if any) that are in the older format. * @param snapshot - The snapshot tree to normalize. * @param config - Configs to use when normalizing snapshot. For example, it can contain paths of blobs whose contents * should be normalized as well. * @returns a copy of the normalized snapshot tree. */ -export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormalizerConfig): ITree { +export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormalizerConfig, depth = 0): ITree { // Merge blobs to normalize in the config with runtime blobs to normalize. The contents of these blobs will be // parsed and deep sorted. const blobsToNormalize = [ ...runtimeBlobsToNormalize, ...config?.blobsToNormalize ?? [] ]; @@ -106,6 +110,10 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali for (const entry of snapshot.entries) { switch (entry.type) { case TreeEntry.Blob: { + if (depth === 0 && entry.path === ".metadata") { + // Ignore the root .metadata blob + continue; + } let contents = entry.value.contents; // If this blob has to be normalized, parse and sort the blob contents first. if (blobsToNormalize.includes(entry.path)) { @@ -115,7 +123,15 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali break; } case TreeEntry.Tree: { - normalizedEntries.push(new TreeTreeEntry(entry.path, getNormalizedSnapshot(entry.value, config))); + const normalizedSubtree = getNormalizedSnapshot(entry.value, config, depth + 1); + if (entry.path === ".channels") { + // These special subtrees get promoted. + for (const subTreeEntry of normalizedSubtree.entries) { + normalizedEntries.push(subTreeEntry); + } + } else { + normalizedEntries.push(new TreeTreeEntry(entry.path, normalizedSubtree)); + } break; } case TreeEntry.Attachment: { From aeef8053e7383ee8b54aaad451a29a3cfba9f6e2 Mon Sep 17 00:00:00 2001 From: Arin Date: Mon, 8 Mar 2021 14:15:36 -0500 Subject: [PATCH 11/35] Fix all broken tests and places where summaryFormatVersion matters --- packages/loader/container-loader/src/utils.ts | 32 +++--- .../container-runtime/src/dataStoreContext.ts | 8 +- .../container-runtime/src/dataStores.ts | 1 + .../src/test/dataStoreContext.spec.ts | 21 ++-- .../src/test/dataStoreCreation.spec.ts | 28 ++++-- .../test/deRehydrateContainerTests.spec.ts | 98 ++++++++++++------- .../src/test/detachedContainerTests.spec.ts | 2 +- 7 files changed, 122 insertions(+), 68 deletions(-) diff --git a/packages/loader/container-loader/src/utils.ts b/packages/loader/container-loader/src/utils.ts index d3e062813e6b..70800928bc12 100644 --- a/packages/loader/container-loader/src/utils.ts +++ b/packages/loader/container-loader/src/utils.ts @@ -33,7 +33,17 @@ export function parseUrl(url: string): IParsedUrl | undefined { : undefined; } -function convertProtocolAndAppSummaryToSnapshotTreeCore( +/** + * Converts summary tree (for upload) to snapshot tree (for download). + * Summary tree blobs contain contents, but snapshot tree blobs normally + * contain IDs pointing to storage. This will create 2 blob entries in the + * snapshot tree for each blob in the summary tree. One will be the regular + * path pointing to a uniquely generated ID. Then there will be another + * entry with the path as that uniquely generated ID, and value as the + * blob contents as a base-64 string. + * @param summary - summary to convert + */ +function convertSummaryToSnapshotWithEmbeddedBlobContents( summary: ISummaryTree, ): ISnapshotTree { const treeNode = { @@ -48,7 +58,7 @@ function convertProtocolAndAppSummaryToSnapshotTreeCore( switch (summaryObject.type) { case SummaryType.Tree: { - treeNode.trees[key] = convertProtocolAndAppSummaryToSnapshotTreeCore(summaryObject); + treeNode.trees[key] = convertSummaryToSnapshotWithEmbeddedBlobContents(summaryObject); break; } case SummaryType.Blob: { @@ -79,20 +89,14 @@ export function convertProtocolAndAppSummaryToSnapshotTree( protocolSummaryTree: ISummaryTree, appSummaryTree: ISummaryTree, ): ISnapshotTree { - const protocolSummaryTreeModified: ISummaryTree = { + // Shallow copy is fine, since we are doing a deep clone below. + const combinedSummary: ISummaryTree = { type: SummaryType.Tree, - tree: { - ".protocol": { - type: SummaryType.Tree, - tree: { ...protocolSummaryTree.tree }, - }, - }, - }; - const snapshotTree = convertProtocolAndAppSummaryToSnapshotTreeCore(protocolSummaryTreeModified); - snapshotTree.trees = { - ...snapshotTree.trees, - ...convertProtocolAndAppSummaryToSnapshotTreeCore(appSummaryTree).trees, + tree: { ...appSummaryTree.tree }, }; + combinedSummary.tree[".protocol"] = protocolSummaryTree; + + const snapshotTree = convertSummaryToSnapshotWithEmbeddedBlobContents(combinedSummary); return snapshotTree; } diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index fb70ff73134b..7cbfb8402c92 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -65,6 +65,7 @@ import { ContainerRuntime } from "./containerRuntime"; import { dataStoreAttributesBlobName, DataStoreSummaryFormatVersion, + summaryFormatVersionToNumber, wrapSummaryInChannelsTree, } from "./summaryFormat"; @@ -792,6 +793,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { private readonly snapshotTree: ISnapshotTree | undefined, protected readonly isRootDataStore: boolean, disableIsolatedChannels: boolean, + protected readonly summaryFormatVersion: DataStoreSummaryFormatVersion, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -861,9 +863,10 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { protected async getInitialSnapshotDetails(): Promise { assert(this.pkg !== undefined, "pkg should be available in local data store"); assert(this.isRootDataStore !== undefined, "isRootDataStore should be available in local data store"); + const versionNumber = summaryFormatVersionToNumber(this.summaryFormatVersion); return { pkg: this.pkg, - snapshot: this.snapshotTree, + snapshot: versionNumber < 2 ? this.snapshotTree : this.snapshotTree?.trees[channelsTreeName], isRootDataStore: this.isRootDataStore, }; } @@ -892,6 +895,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { snapshotTree: ISnapshotTree | undefined, isRootDataStore: boolean, disableIsolatedChannels: boolean, + summaryFormatVersion: DataStoreSummaryFormatVersion, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -908,6 +912,7 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { snapshotTree, isRootDataStore, disableIsolatedChannels, + summaryFormatVersion, createProps); } } @@ -945,6 +950,7 @@ export class LocalDetachedFluidDataStoreContext snapshotTree, isRootDataStore, disableIsolatedChannels, + disableIsolatedChannels ? "0.1" : 2, ); this.detachedRuntimeCreation = true; } diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index 61b0c9ca1ce3..6473c57ab527 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -138,6 +138,7 @@ export class DataStores implements IDisposable { // roots in the document but they won't break. attributes.isRootDataStore ?? true, this.disableIsolatedChannels, + attributes.snapshotFormatVersion, ); } this.contexts.addBoundOrRemoted(dataStoreContext); diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index 0cc03e4d61d2..f6a85bca7da7 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -100,7 +100,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, true /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); await localDataStoreContext.realize(); const attachMessage = localDataStoreContext.generateAttachMessage(); @@ -139,7 +140,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); await localDataStoreContext.realize() .catch((error) => { @@ -172,7 +174,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); await localDataStoreContext.realize(); @@ -210,7 +213,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, true /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, true, "The data store should be root."); @@ -227,7 +231,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, false, "The data store should not be root."); @@ -246,7 +251,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, true /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); const gcData = await localDataStoreContext.getGCData(); assert.deepStrictEqual(gcData, emptyGCData, "GC data from getGCData should be empty."); @@ -272,7 +278,8 @@ describe("Data Store Context Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); // Get the summarizer node for this data store which tracks its referenced state. const dataStoreSummarizerNode = summarizerNode.getChild(dataStoreId); diff --git a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts index a43df5041abd..a68fe5b345b9 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts @@ -122,7 +122,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await context.realize(); @@ -147,7 +148,9 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2, + 2); try { await context.realize(); @@ -172,7 +175,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextA.realize(); @@ -197,7 +201,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextB.realize(); @@ -222,7 +227,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextB.realize(); @@ -244,7 +250,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextC.realize(); @@ -269,7 +276,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextFake.realize(); @@ -294,7 +302,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextFake.realize(); @@ -319,7 +328,8 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */); + false /* disableIsolatedChannels */, + 2); try { await contextC.realize(); diff --git a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 25d5b3095504..c9bb3861b7ea 100644 --- a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -15,7 +15,7 @@ import { LoaderContainerTracker, } from "@fluidframework/test-utils"; import { SharedMap, SharedDirectory } from "@fluidframework/map"; -import { IDocumentAttributes } from "@fluidframework/protocol-definitions"; +import { IDocumentAttributes, ISnapshotTree } from "@fluidframework/protocol-definitions"; import { IContainerRuntimeBase } from "@fluidframework/runtime-definitions"; import { ConsensusRegisterCollection } from "@fluidframework/register-collection"; import { SharedString, SparseMatrix } from "@fluidframework/sequence"; @@ -30,6 +30,38 @@ import { ITestDriver } from "@fluidframework/test-driver-definitions"; const detachedContainerRefSeqNumber = 0; +function assertSubtree(tree: ISnapshotTree, key: string, msg?: string): ISnapshotTree { + const subTree = tree.trees[key]; + assert(subTree, msg ?? `${key} subtree not present`); + return subTree; +} + +const assertChannelsTree = (root: ISnapshotTree) => assertSubtree(root, ".channels"); +const assertProtocolTree = (root: ISnapshotTree) => assertSubtree(root, ".protocol"); + +function assertChannelTree(rootOrDatastore: ISnapshotTree, key: string, msg?: string) { + const channelsTree = assertChannelsTree(rootOrDatastore); + return { + channelsTree, + datastoreTree: assertSubtree(channelsTree, key, msg ?? `${key} channel not present`), + }; +} +const assertDatastoreTree = (root: ISnapshotTree, key: string, msg?: string) => + assertChannelTree(root, key, `${key} datastore not present`); + +const assertSchedulerTree = (root: ISnapshotTree) => assertDatastoreTree(root, "_scheduler"); + +function assertBlobContents(subtree: ISnapshotTree, key: string): T { + const id = subtree.blobs[key]; + assert(id, `blob id for ${key} missing`); + const contents = subtree.blobs[id]; + assert(contents, `blob contents for ${key} missing`); + return JSON.parse(fromBase64ToUtf8(contents)) as T; +} + +const assertProtocolAttributes = (s: ISnapshotTree) => + assertBlobContents(assertProtocolTree(s), "attributes"); + describe(`Dehydrate Rehydrate Container Test`, () => { let driver: ITestDriver; before(()=>{ @@ -115,58 +147,52 @@ describe(`Dehydrate Rehydrate Container Test`, () => { it("Dehydrated container snapshot", async () => { const { container } = await createDetachedContainerAndGetRootDataStore(); - const snapshotTree = JSON.parse(container.serialize()); + const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); - assert.ok(snapshotTree.trees[".protocol"], "protocol tree not present"); - assert.ok(snapshotTree.trees.default, "default dataStore tree not present"); - assert.ok(snapshotTree.trees._scheduler, "scheduler tree not present"); - assert.strictEqual(Object.keys(snapshotTree.trees[".protocol"].blobs).length, 8, - "4 protocol blobs should be there(8 mappings)"); + // Check for scheduler + assertSchedulerTree(snapshotTree); // Check for protocol attributes - const protocolAttributesBlobId = snapshotTree.trees[".protocol"].blobs.attributes; - const protocolAttributes: IDocumentAttributes = - JSON.parse(fromBase64ToUtf8(snapshotTree.trees[".protocol"].blobs[protocolAttributesBlobId])); + const protocolTree = assertProtocolTree(snapshotTree); + assert.strictEqual(Object.keys(protocolTree.blobs).length, 8, + "4 protocol blobs should be there(8 mappings)"); + + const protocolAttributes = assertProtocolAttributes(snapshotTree); assert.strictEqual(protocolAttributes.sequenceNumber, detachedContainerRefSeqNumber, "initial aeq #"); assert( protocolAttributes.minimumSequenceNumber <= protocolAttributes.sequenceNumber, "Min Seq # <= seq #"); // Check for default dataStore - const defaultDataStoreBlobId = snapshotTree.trees.default.blobs[".component"]; - const dataStoreAttributes = - JSON.parse(fromBase64ToUtf8(snapshotTree.trees.default.blobs[defaultDataStoreBlobId])); - assert.strictEqual(dataStoreAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); + const { datastoreTree: defaultDatastore } = assertDatastoreTree(snapshotTree, "default"); + const datastoreAttributes = assertBlobContents<{ pkg: string }>(defaultDatastore, ".component"); + assert.strictEqual(datastoreAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); }); it("Dehydrated container snapshot 2 times with changes in between", async () => { const { container, defaultDataStore } = await createDetachedContainerAndGetRootDataStore(); - const snapshotTree1 = JSON.parse(container.serialize()); + const snapshotTree1: ISnapshotTree = JSON.parse(container.serialize()); // Create a channel const channel = defaultDataStore.runtime.createChannel("test1", "https://graph.microsoft.com/types/map") as SharedMap; channel.bindToContext(); - const snapshotTree2 = JSON.parse(container.serialize()); + const snapshotTree2: ISnapshotTree = JSON.parse(container.serialize()); assert.strictEqual(JSON.stringify(Object.keys(snapshotTree1.trees)), JSON.stringify(Object.keys(snapshotTree2.trees)), "3 trees should be there(protocol, default dataStore, scheduler"); // Check for protocol attributes - const protocolAttributesBlobId1 = snapshotTree1.trees[".protocol"].blobs.attributes; - const protocolAttributesBlobId2 = snapshotTree2.trees[".protocol"].blobs.attributes; - const protocolAttributes1: IDocumentAttributes = - JSON.parse(fromBase64ToUtf8(snapshotTree1.trees[".protocol"].blobs[protocolAttributesBlobId1])); - const protocolAttributes2: IDocumentAttributes = - JSON.parse(fromBase64ToUtf8(snapshotTree2.trees[".protocol"].blobs[protocolAttributesBlobId2])); + const protocolAttributes1 = assertProtocolAttributes(snapshotTree1); + const protocolAttributes2 = assertProtocolAttributes(snapshotTree2); assert.strictEqual(JSON.stringify(protocolAttributes1), JSON.stringify(protocolAttributes2), "Protocol attributes should be same as no change happened"); // Check for newly create channel - assert.strictEqual(snapshotTree1.trees.default.trees.test1, undefined, - "Test channel 1 should not be present in snapshot 1"); - assert(snapshotTree2.trees.default.trees.test1, + const defaultChannelsTree1 = assertChannelsTree(assertDatastoreTree(snapshotTree1, "default").datastoreTree); + assert(defaultChannelsTree1.trees.test1 === undefined, "Test channel 1 should not be present in snapshot 1"); + assertChannelTree(assertDatastoreTree(snapshotTree2, "default").datastoreTree, "test1", "Test channel 1 should be present in snapshot 2"); }); @@ -182,18 +208,18 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); rootOfDataStore1.set("dataStore2", dataStore2.handle); - const snapshotTree = JSON.parse(container.serialize()); + const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); - assert.ok(snapshotTree.trees[".protocol"], "protocol tree not present"); - assert.ok(snapshotTree.trees.default, "default dataStore tree not present"); - assert.ok(snapshotTree.trees._scheduler, "scheduler tree not present"); - assert.ok( + assertProtocolTree(snapshotTree); + const { channelsTree } = assertSchedulerTree(snapshotTree); + assertDatastoreTree(snapshotTree, "default"); + assert( // eslint-disable-next-line unicorn/no-unsafe-regex - Object.keys(snapshotTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), + Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), "peer data store tree not present", ); - assert(snapshotTree.trees[dataStore2.runtime.id], "Handle Bounded dataStore should be in summary"); + assertDatastoreTree(snapshotTree, dataStore2.runtime.id, "Handle Bounded dataStore should be in summary"); }); it("Rehydrate container from snapshot and check contents before attach", async () => { @@ -546,12 +572,12 @@ describe(`Dehydrate Rehydrate Container Test`, () => { const snapshotTree = JSON.parse(container.serialize()); - assert.ok(snapshotTree.trees[".protocol"], "protocol tree not present"); - assert.ok(snapshotTree.trees.default, "default dataStore tree not present"); - assert.ok(snapshotTree.trees._scheduler, "scheduler tree not present"); + assertProtocolTree(snapshotTree); + const { channelsTree } = assertSchedulerTree(snapshotTree); + assertDatastoreTree(snapshotTree, "default"); assert.ok( // eslint-disable-next-line unicorn/no-unsafe-regex - !Object.keys(snapshotTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), + !Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), "unbounded/unreferenced data store tree present", ); }); diff --git a/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts index 367cbf333c83..235f4cb36565 100644 --- a/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -576,7 +576,7 @@ const tests = (argsFactory: () => ITestObjectProvider) => { await defPromise.promise; }); - it.skip("Fire ops during container attach for shared matrix", async () => { + it("Fire ops during container attach for shared matrix", async () => { const op = { pos1: 0, seg: 9, type: 0, target: "rows" }; const defPromise = new Deferred(); const container = await loader.createDetachedContainer(args.defaultCodeDetails); From 2f588303ca03a6f6e75cca992b29155ecedfe6c7 Mon Sep 17 00:00:00 2001 From: Arin Date: Mon, 8 Mar 2021 16:25:37 -0500 Subject: [PATCH 12/35] Final fixes --- .../runtime/container-runtime/src/containerRuntime.ts | 9 ++++++++- .../test-end-to-end-tests/src/test/summaries.spec.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 86411611aef1..bf462bda2317 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -231,6 +231,7 @@ export interface IContainerRuntimeOptions { // Flag that disables putting channels in isolated subtrees for each data store // and the root node when generating a summary if set to true. + // Defaults to TRUE (disabled) for now. disableIsolatedChannels?: boolean; } @@ -669,13 +670,14 @@ export class ContainerRuntime extends TypedEventEmitter private readonly chunkMap: Map; private readonly dataStores: DataStores; + private readonly runtimeOptions: Readonly; private constructor( private readonly context: IContainerContext, private readonly registry: IFluidDataStoreRegistry, metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: undefined }, chunks: [string, string[]][], - private readonly runtimeOptions: IContainerRuntimeOptions = { + runtimeOptions: IContainerRuntimeOptions = { generateSummaries: true, }, private readonly containerScope: IFluidObject, @@ -685,6 +687,11 @@ export class ContainerRuntime extends TypedEventEmitter ) { super(); + this.runtimeOptions = { + ...{ disableIsolatedChannels: false }, + ...runtimeOptions, + }; + this._connected = this.context.connected; this.chunkMap = new Map(chunks); diff --git a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts index ab8e7098ce6f..e0b5edfdc30b 100644 --- a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts @@ -84,7 +84,7 @@ describe("Summaries", () => { await opProcessingController.process(); const { gcData, stats, summary } = await containerRuntime.summarize({ - runGc: false, + runGC: false, fullTree: false, trackState: false, summaryLogger: new TelemetryNullLogger(), From b393dc7e04d4d9487928198b16f2ba8fa4f99a29 Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 9 Mar 2021 03:16:40 -0500 Subject: [PATCH 13/35] Update tests --- .../container-runtime/src/containerRuntime.ts | 4 +- .../test/deRehydrateContainerTests.spec.ts | 886 +++++++++--------- .../src/test/summaries.spec.ts | 77 +- .../test/test-utils/src/localCodeLoader.ts | 12 +- 4 files changed, 531 insertions(+), 448 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index bf462bda2317..86a8e61adcb0 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -951,7 +951,9 @@ export class ContainerRuntime extends TypedEventEmitter public async snapshot(): Promise { const root: ITree = { entries: [] }; - if (!this.runtimeOptions.disableIsolatedChannels) { + if (this.runtimeOptions.disableIsolatedChannels) { + root.entries = root.entries.concat(await this.dataStores.snapshot()); + } else { root.entries.push(new TreeTreeEntry( channelsTreeName, { entries: await this.dataStores.snapshot() }, diff --git a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index c9bb3861b7ea..4d95e7e96683 100644 --- a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -30,44 +30,47 @@ import { ITestDriver } from "@fluidframework/test-driver-definitions"; const detachedContainerRefSeqNumber = 0; -function assertSubtree(tree: ISnapshotTree, key: string, msg?: string): ISnapshotTree { - const subTree = tree.trees[key]; - assert(subTree, msg ?? `${key} subtree not present`); - return subTree; -} - -const assertChannelsTree = (root: ISnapshotTree) => assertSubtree(root, ".channels"); -const assertProtocolTree = (root: ISnapshotTree) => assertSubtree(root, ".protocol"); - -function assertChannelTree(rootOrDatastore: ISnapshotTree, key: string, msg?: string) { - const channelsTree = assertChannelsTree(rootOrDatastore); - return { - channelsTree, - datastoreTree: assertSubtree(channelsTree, key, msg ?? `${key} channel not present`), - }; -} -const assertDatastoreTree = (root: ISnapshotTree, key: string, msg?: string) => - assertChannelTree(root, key, `${key} datastore not present`); - -const assertSchedulerTree = (root: ISnapshotTree) => assertDatastoreTree(root, "_scheduler"); - -function assertBlobContents(subtree: ISnapshotTree, key: string): T { - const id = subtree.blobs[key]; - assert(id, `blob id for ${key} missing`); - const contents = subtree.blobs[id]; - assert(contents, `blob contents for ${key} missing`); - return JSON.parse(fromBase64ToUtf8(contents)) as T; -} - -const assertProtocolAttributes = (s: ISnapshotTree) => - assertBlobContents(assertProtocolTree(s), "attributes"); - describe(`Dehydrate Rehydrate Container Test`, () => { + let disableIsolatedChannels = false; let driver: ITestDriver; before(()=>{ driver = getFluidTestDriver() as unknown as ITestDriver; }); + function assertSubtree(tree: ISnapshotTree, key: string, msg?: string): ISnapshotTree { + const subTree = tree.trees[key]; + assert(subTree, msg ?? `${key} subtree not present`); + return subTree; + } + + const assertChannelsTree = (rootOrDatastore: ISnapshotTree) => disableIsolatedChannels + ? rootOrDatastore + : assertSubtree(rootOrDatastore, ".channels"); + const assertProtocolTree = (root: ISnapshotTree) => assertSubtree(root, ".protocol"); + + function assertChannelTree(rootOrDatastore: ISnapshotTree, key: string, msg?: string) { + const channelsTree = assertChannelsTree(rootOrDatastore); + return { + channelsTree, + datastoreTree: assertSubtree(channelsTree, key, msg ?? `${key} channel not present`), + }; + } + const assertDatastoreTree = (root: ISnapshotTree, key: string, msg?: string) => + assertChannelTree(root, key, `${key} datastore not present`); + + const assertSchedulerTree = (root: ISnapshotTree) => assertDatastoreTree(root, "_scheduler"); + + function assertBlobContents(subtree: ISnapshotTree, key: string): T { + const id = subtree.blobs[key]; + assert(id, `blob id for ${key} missing`); + const contents = subtree.blobs[id]; + assert(contents, `blob contents for ${key} missing`); + return JSON.parse(fromBase64ToUtf8(contents)) as T; + } + + const assertProtocolAttributes = (s: ISnapshotTree) => + assertBlobContents(assertProtocolTree(s), "attributes"); + const codeDetails: IFluidCodeDetails = { package: "detachedContainerTestPackage1", config: {}, @@ -111,7 +114,7 @@ describe(`Dehydrate Rehydrate Container Test`, () => { [sparseMatrixId, SparseMatrix.getFactory()], [sharedCounterId, SharedCounter.getFactory()], ]); - const codeLoader = new LocalCodeLoader([[codeDetails, factory]]); + const codeLoader = new LocalCodeLoader([[codeDetails, factory]], { disableIsolatedChannels }); const documentServiceFactory = driver.createDocumentServiceFactory(); const testLoader = new Loader({ urlResolver: driver.createUrlResolver(), @@ -144,441 +147,454 @@ describe(`Dehydrate Rehydrate Container Test`, () => { loaderContainerTracker.reset(); }); - it("Dehydrated container snapshot", async () => { - const { container } = - await createDetachedContainerAndGetRootDataStore(); - const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); - - // Check for scheduler - assertSchedulerTree(snapshotTree); - - // Check for protocol attributes - const protocolTree = assertProtocolTree(snapshotTree); - assert.strictEqual(Object.keys(protocolTree.blobs).length, 8, - "4 protocol blobs should be there(8 mappings)"); - - const protocolAttributes = assertProtocolAttributes(snapshotTree); - assert.strictEqual(protocolAttributes.sequenceNumber, detachedContainerRefSeqNumber, "initial aeq #"); - assert( - protocolAttributes.minimumSequenceNumber <= protocolAttributes.sequenceNumber, - "Min Seq # <= seq #"); - - // Check for default dataStore - const { datastoreTree: defaultDatastore } = assertDatastoreTree(snapshotTree, "default"); - const datastoreAttributes = assertBlobContents<{ pkg: string }>(defaultDatastore, ".component"); - assert.strictEqual(datastoreAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); - }); - - it("Dehydrated container snapshot 2 times with changes in between", async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - const snapshotTree1: ISnapshotTree = JSON.parse(container.serialize()); - // Create a channel - const channel = defaultDataStore.runtime.createChannel("test1", - "https://graph.microsoft.com/types/map") as SharedMap; - channel.bindToContext(); - const snapshotTree2: ISnapshotTree = JSON.parse(container.serialize()); - - assert.strictEqual(JSON.stringify(Object.keys(snapshotTree1.trees)), - JSON.stringify(Object.keys(snapshotTree2.trees)), - "3 trees should be there(protocol, default dataStore, scheduler"); - - // Check for protocol attributes - const protocolAttributes1 = assertProtocolAttributes(snapshotTree1); - const protocolAttributes2 = assertProtocolAttributes(snapshotTree2); - assert.strictEqual(JSON.stringify(protocolAttributes1), JSON.stringify(protocolAttributes2), - "Protocol attributes should be same as no change happened"); - - // Check for newly create channel - const defaultChannelsTree1 = assertChannelsTree(assertDatastoreTree(snapshotTree1, "default").datastoreTree); - assert(defaultChannelsTree1.trees.test1 === undefined, "Test channel 1 should not be present in snapshot 1"); - assertChannelTree(assertDatastoreTree(snapshotTree2, "default").datastoreTree, "test1", - "Test channel 1 should be present in snapshot 2"); - }); - - it("Dehydrated container snapshot with dataStore handle stored in map of other bound dataStore", async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - - // Create another dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - - // Create a channel - const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); - rootOfDataStore1.set("dataStore2", dataStore2.handle); - - const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); + const tests = () => { + it("Dehydrated container snapshot", async () => { + const { container } = + await createDetachedContainerAndGetRootDataStore(); + const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); + + // Check for scheduler + assertSchedulerTree(snapshotTree); + + // Check for protocol attributes + const protocolTree = assertProtocolTree(snapshotTree); + assert.strictEqual(Object.keys(protocolTree.blobs).length, 8, + "4 protocol blobs should be there(8 mappings)"); + + const protocolAttributes = assertProtocolAttributes(snapshotTree); + assert.strictEqual(protocolAttributes.sequenceNumber, detachedContainerRefSeqNumber, "initial aeq #"); + assert( + protocolAttributes.minimumSequenceNumber <= protocolAttributes.sequenceNumber, + "Min Seq # <= seq #"); + + // Check for default dataStore + const { datastoreTree: defaultDatastore } = assertDatastoreTree(snapshotTree, "default"); + const datastoreAttributes = assertBlobContents<{ pkg: string }>(defaultDatastore, ".component"); + assert.strictEqual(datastoreAttributes.pkg, JSON.stringify(["default"]), "Package name should be default"); + }); - assertProtocolTree(snapshotTree); - const { channelsTree } = assertSchedulerTree(snapshotTree); - assertDatastoreTree(snapshotTree, "default"); - assert( - // eslint-disable-next-line unicorn/no-unsafe-regex - Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), - "peer data store tree not present", - ); + it("Dehydrated container snapshot 2 times with changes in between", async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); + const snapshotTree1: ISnapshotTree = JSON.parse(container.serialize()); + // Create a channel + const channel = defaultDataStore.runtime.createChannel("test1", + "https://graph.microsoft.com/types/map") as SharedMap; + channel.bindToContext(); + const snapshotTree2: ISnapshotTree = JSON.parse(container.serialize()); + + assert.strictEqual(JSON.stringify(Object.keys(snapshotTree1.trees)), + JSON.stringify(Object.keys(snapshotTree2.trees)), + "3 trees should be there(protocol, default dataStore, scheduler"); + + // Check for protocol attributes + const protocolAttributes1 = assertProtocolAttributes(snapshotTree1); + const protocolAttributes2 = assertProtocolAttributes(snapshotTree2); + assert.strictEqual(JSON.stringify(protocolAttributes1), JSON.stringify(protocolAttributes2), + "Protocol attributes should be same as no change happened"); + + // Check for newly create channel + const defaultChannelsTree1 = assertChannelsTree( + assertDatastoreTree(snapshotTree1, "default").datastoreTree); + assert(defaultChannelsTree1.trees.test1 === undefined, + "Test channel 1 should not be present in snapshot 1"); + assertChannelTree(assertDatastoreTree(snapshotTree2, "default").datastoreTree, "test1", + "Test channel 1 should be present in snapshot 2"); + }); - assertDatastoreTree(snapshotTree, dataStore2.runtime.id, "Handle Bounded dataStore should be in summary"); - }); + it("Dehydrated container snapshot with dataStore handle stored in map of other bound dataStore", async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); - it("Rehydrate container from snapshot and check contents before attach", async () => { - const { container } = - await createDetachedContainerAndGetRootDataStore(); + // Create another dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - const snapshotTree = container.serialize(); + // Create a channel + const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); + rootOfDataStore1.set("dataStore2", dataStore2.handle); - const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + const snapshotTree: ISnapshotTree = JSON.parse(container.serialize()); - // Check for scheduler - const schedulerResponse = await container2.request({ url: "_scheduler" }); - assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerDataStore = schedulerResponse.value as TestFluidObject; - assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); + assertProtocolTree(snapshotTree); + const { channelsTree } = assertSchedulerTree(snapshotTree); + assertDatastoreTree(snapshotTree, "default"); + assert( + // eslint-disable-next-line unicorn/no-unsafe-regex + Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), + "peer data store tree not present", + ); - // Check for default data store - const response = await container2.request({ url: "/" }); - assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultDataStore = response.value as TestFluidObject; - assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); - - // Check for dds - const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); - const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); - const sharedString = await defaultDataStore.getSharedObject(sharedStringId); - const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); - const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); - const crc = await defaultDataStore.getSharedObject>(crcId); - const coc = await defaultDataStore.getSharedObject(cocId); - const ink = await defaultDataStore.getSharedObject(sharedInkId); - const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); - assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); - assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); - assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); - assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); - assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); - assert.strictEqual(crc.id, crcId, "CRC should exist!!"); - assert.strictEqual(coc.id, cocId, "COC should exist!!"); - assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); - assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); - assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); - }); - - it("Rehydrate container from snapshot and check contents after attach", async () => { - const { container } = - await createDetachedContainerAndGetRootDataStore(); + assertDatastoreTree(snapshotTree, dataStore2.runtime.id, "Handle Bounded dataStore should be in summary"); + }); - const snapshotTree = container.serialize(); + it("Rehydrate container from snapshot and check contents before attach", async () => { + const { container } = + await createDetachedContainerAndGetRootDataStore(); + + const snapshotTree = container.serialize(); + + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + + // Check for scheduler + const schedulerResponse = await container2.request({ url: "_scheduler" }); + assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); + const schedulerDataStore = schedulerResponse.value as TestFluidObject; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); + + // Check for default data store + const response = await container2.request({ url: "/" }); + assert.strictEqual(response.status, 200, "Component should exist!!"); + const defaultDataStore = response.value as TestFluidObject; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); + + // Check for dds + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); + assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); + assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); + assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); + assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); + assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); + assert.strictEqual(crc.id, crcId, "CRC should exist!!"); + assert.strictEqual(coc.id, cocId, "COC should exist!!"); + assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); + assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); + assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); + }); - const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - await container2.attach(request); + it("Rehydrate container from snapshot and check contents after attach", async () => { + const { container } = + await createDetachedContainerAndGetRootDataStore(); + + const snapshotTree = container.serialize(); + + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + await container2.attach(request); + + // Check for scheduler + const schedulerResponse = await container2.request({ url: "_scheduler" }); + assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); + const schedulerDataStore = schedulerResponse.value as TestFluidObject; + assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); + + // Check for default data store + const response = await container2.request({ url: "/" }); + assert.strictEqual(response.status, 200, "Component should exist!!"); + const defaultDataStore = response.value as TestFluidObject; + assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); + + // Check for dds + const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); + const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); + const sharedString = await defaultDataStore.getSharedObject(sharedStringId); + const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); + const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); + const crc = await defaultDataStore.getSharedObject>(crcId); + const coc = await defaultDataStore.getSharedObject(cocId); + const ink = await defaultDataStore.getSharedObject(sharedInkId); + const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); + const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); + assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); + assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); + assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); + assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); + assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); + assert.strictEqual(crc.id, crcId, "CRC should exist!!"); + assert.strictEqual(coc.id, cocId, "COC should exist!!"); + assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); + assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); + assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); + }); - // Check for scheduler - const schedulerResponse = await container2.request({ url: "_scheduler" }); - assert.strictEqual(schedulerResponse.status, 200, "Scheduler Component should exist!!"); - const schedulerDataStore = schedulerResponse.value as TestFluidObject; - assert.strictEqual(schedulerDataStore.runtime.id, "_scheduler", "Id should be of scheduler"); + it("Change contents of dds, then rehydrate and then check summary", async () => { + const { container } = + await createDetachedContainerAndGetRootDataStore(); - // Check for default data store - const response = await container2.request({ url: "/" }); - assert.strictEqual(response.status, 200, "Component should exist!!"); - const defaultDataStore = response.value as TestFluidObject; - assert.strictEqual(defaultDataStore.runtime.id, "default", "Id should be default"); - - // Check for dds - const sharedMap = await defaultDataStore.getSharedObject(sharedMapId); - const sharedDir = await defaultDataStore.getSharedObject(sharedDirectoryId); - const sharedString = await defaultDataStore.getSharedObject(sharedStringId); - const sharedCell = await defaultDataStore.getSharedObject(sharedCellId); - const sharedCounter = await defaultDataStore.getSharedObject(sharedCounterId); - const crc = await defaultDataStore.getSharedObject>(crcId); - const coc = await defaultDataStore.getSharedObject(cocId); - const ink = await defaultDataStore.getSharedObject(sharedInkId); - const sharedMatrix = await defaultDataStore.getSharedObject(sharedMatrixId); - const sparseMatrix = await defaultDataStore.getSharedObject(sparseMatrixId); - assert.strictEqual(sharedMap.id, sharedMapId, "Shared map should exist!!"); - assert.strictEqual(sharedDir.id, sharedDirectoryId, "Shared directory should exist!!"); - assert.strictEqual(sharedString.id, sharedStringId, "Shared string should exist!!"); - assert.strictEqual(sharedCell.id, sharedCellId, "Shared cell should exist!!"); - assert.strictEqual(sharedCounter.id, sharedCounterId, "Shared counter should exist!!"); - assert.strictEqual(crc.id, crcId, "CRC should exist!!"); - assert.strictEqual(coc.id, cocId, "COC should exist!!"); - assert.strictEqual(ink.id, sharedInkId, "Shared ink should exist!!"); - assert.strictEqual(sharedMatrix.id, sharedMatrixId, "Shared matrix should exist!!"); - assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); - }); + const responseBefore = await container.request({ url: "/" }); + const defaultDataStoreBefore = responseBefore.value as TestFluidObject; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); + sharedStringBefore.insertText(0, "Hello"); - it("Change contents of dds, then rehydrate and then check summary", async () => { - const { container } = - await createDetachedContainerAndGetRootDataStore(); + const snapshotTree = container.serialize(); - const responseBefore = await container.request({ url: "/" }); - const defaultDataStoreBefore = responseBefore.value as TestFluidObject; - const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); - sharedStringBefore.insertText(0, "Hello"); + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - const snapshotTree = container.serialize(); + const responseAfter = await container2.request({ url: "/" }); + const defaultComponentAfter = responseAfter.value as TestFluidObject; + const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); + assert.strictEqual( + JSON.stringify(sharedStringAfter.summarize()), + JSON.stringify(sharedStringBefore.summarize()), + "Summaries of shared string should match and contents should be same!!"); + }); - const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + it("Rehydrate container from summary, change contents of dds and then check summary", async () => { + const { container } = + await createDetachedContainerAndGetRootDataStore(); + let str = "AA"; + const response1 = await container.request({ url: "/" }); + const defaultComponent1 = response1.value as TestFluidObject; + const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); + sharedString1.insertText(0, str); + const snapshotTree = container.serialize(); + + const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + const responseBefore = await container2.request({ url: "/" }); + const defaultDataStoreBefore = responseBefore.value as TestFluidObject; + const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); + const sharedMapBefore = await defaultDataStoreBefore.getSharedObject(sharedMapId); + str += "BB"; + sharedStringBefore.insertText(0, str); + sharedMapBefore.set("0", str); + + await container2.attach(request); + const responseAfter = await container2.request({ url: "/" }); + const defaultComponentAfter = responseAfter.value as TestFluidObject; + const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); + const sharedMapAfter = await defaultComponentAfter.getSharedObject(sharedMapId); + assert.strictEqual( + JSON.stringify(sharedStringAfter.summarize()), + JSON.stringify(sharedStringBefore.summarize()), + "Summaries of shared string should match and contents should be same!!"); + assert.strictEqual( + JSON.stringify(sharedMapAfter.summarize()), + JSON.stringify(sharedMapBefore.summarize()), + "Summaries of shared map should match and contents should be same!!"); + }); - const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as TestFluidObject; - const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); - assert.strictEqual( - JSON.stringify(sharedStringAfter.summarize()), - JSON.stringify(sharedStringBefore.summarize()), - "Summaries of shared string should match and contents should be same!!"); - }); + it("Rehydrate container, don't load a data store and then load after container attachment. Make changes to " + + "dds from rehydrated container and check reflection of changes in other container", + async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); + + // Create another dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; + peerDataStore.peerDataStoreRuntimeChannel.bindToContext(); + const sharedMap1 = await dataStore2.getSharedObject(sharedMapId); + sharedMap1.set("0", "A"); + const snapshotTree = container.serialize(); + // close the container that we don't use any more, so it doesn't block ensureSynchronized() + container.close(); + + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + await rehydratedContainer.attach(request); + + // Now load the container from another loader. + const urlResolver2 = driver.createUrlResolver(); + const loader2 = createTestLoader(); + assert(rehydratedContainer.resolvedUrl); + const requestUrl2 = await urlResolver2.getAbsoluteUrl(rehydratedContainer.resolvedUrl, ""); + const container2 = await loader2.resolve({ url: requestUrl2 }); + + // Get the sharedString1 from dataStore2 in rehydrated container. + const responseBefore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); + const dataStore2FromRC = responseBefore.value as TestFluidObject; + const sharedMapFromRC = await dataStore2FromRC.getSharedObject(sharedMapId); + sharedMapFromRC.set("1", "B"); + + const responseAfter = await container2.request({ url: `/${dataStore2.context.id}` }); + const dataStore3 = responseAfter.value as TestFluidObject; + const sharedMap3 = await dataStore3.getSharedObject(sharedMapId); + + await loaderContainerTracker.ensureSynchronized(); + assert.strictEqual(sharedMap3.get("1"), "B", "Contents should be as required"); + assert.strictEqual( + JSON.stringify(sharedMap3.summarize()), + JSON.stringify(sharedMapFromRC.summarize()), + "Summaries of shared string should match and contents should be same!!"); + }); - it("Rehydrate container from summary, change contents of dds and then check summary", async () => { - const { container } = - await createDetachedContainerAndGetRootDataStore(); - let str = "AA"; - const response1 = await container.request({ url: "/" }); - const defaultComponent1 = response1.value as TestFluidObject; - const sharedString1 = await defaultComponent1.getSharedObject(sharedStringId); - sharedString1.insertText(0, str); - const snapshotTree = container.serialize(); - - const container2 = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - const responseBefore = await container2.request({ url: "/" }); - const defaultDataStoreBefore = responseBefore.value as TestFluidObject; - const sharedStringBefore = await defaultDataStoreBefore.getSharedObject(sharedStringId); - const sharedMapBefore = await defaultDataStoreBefore.getSharedObject(sharedMapId); - str += "BB"; - sharedStringBefore.insertText(0, str); - sharedMapBefore.set("0", str); - - await container2.attach(request); - const responseAfter = await container2.request({ url: "/" }); - const defaultComponentAfter = responseAfter.value as TestFluidObject; - const sharedStringAfter = await defaultComponentAfter.getSharedObject(sharedStringId); - const sharedMapAfter = await defaultComponentAfter.getSharedObject(sharedMapId); - assert.strictEqual( - JSON.stringify(sharedStringAfter.summarize()), - JSON.stringify(sharedStringBefore.summarize()), - "Summaries of shared string should match and contents should be same!!"); - assert.strictEqual( - JSON.stringify(sharedMapAfter.summarize()), - JSON.stringify(sharedMapBefore.summarize()), - "Summaries of shared map should match and contents should be same!!"); - }); + it("Rehydrate container, create but don't load a data store. Attach rehydrated container and load " + + "container 2 from another loader. Then load the created dataStore from container 2, make changes to dds " + + "in it check reflection of changes in rehydrated container", + async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); + + // Create another dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; + peerDataStore.peerDataStoreRuntimeChannel.bindToContext(); + const sharedMap1 = await dataStore2.getSharedObject(sharedMapId); + sharedMap1.set("0", "A"); + const snapshotTree = container.serialize(); + // close the container that we don't use any more, so it doesn't block ensureSynchronized() + container.close(); + + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + await rehydratedContainer.attach(request); + + // Now load the container from another loader. + const urlResolver2 = driver.createUrlResolver(); + const loader2 = createTestLoader(); + assert(rehydratedContainer.resolvedUrl); + const requestUrl2 = await urlResolver2.getAbsoluteUrl(rehydratedContainer.resolvedUrl, ""); + const container2 = await loader2.resolve({ url: requestUrl2 }); + + // Get the sharedString1 from dataStore2 in container2. + const responseBefore = await container2.request({ url: `/${dataStore2.context.id}` }); + const dataStore3 = responseBefore.value as TestFluidObject; + const sharedMap3 = await dataStore3.getSharedObject(sharedMapId); + sharedMap3.set("1", "B"); + + // Get the sharedString1 from dataStore2 in rehydrated container. + const responseAfter = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); + const dataStore2FromRC = responseAfter.value as TestFluidObject; + const sharedMapFromRC = await dataStore2FromRC.getSharedObject(sharedMapId); + + await loaderContainerTracker.ensureSynchronized(); + assert.strictEqual(sharedMapFromRC.get("1"), "B", "Changes should be reflected in other map"); + assert.strictEqual( + JSON.stringify(sharedMap3.summarize()), + JSON.stringify(sharedMapFromRC.summarize()), + "Summaries of shared string should match and contents should be same!!"); + }); - it("Rehydrate container, don't load a data store and then load after container attachment. Make changes to " + - "dds from rehydrated container and check reflection of changes in other container", - async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - - // Create another dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - peerDataStore.peerDataStoreRuntimeChannel.bindToContext(); - const sharedMap1 = await dataStore2.getSharedObject(sharedMapId); - sharedMap1.set("0", "A"); - const snapshotTree = container.serialize(); - container.close(); // close the container that we don't use any more, so it doesn't block ensureSynchronized() - - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - await rehydratedContainer.attach(request); - - // Now load the container from another loader. - const urlResolver2 = driver.createUrlResolver(); - const loader2 = createTestLoader(); - assert(rehydratedContainer.resolvedUrl); - const requestUrl2 = await urlResolver2.getAbsoluteUrl(rehydratedContainer.resolvedUrl, ""); - const container2 = await loader2.resolve({ url: requestUrl2 }); - - // Get the sharedString1 from dataStore2 in rehydrated container. - const responseBefore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); - const dataStore2FromRC = responseBefore.value as TestFluidObject; - const sharedMapFromRC = await dataStore2FromRC.getSharedObject(sharedMapId); - sharedMapFromRC.set("1", "B"); - - const responseAfter = await container2.request({ url: `/${dataStore2.context.id}` }); - const dataStore3 = responseAfter.value as TestFluidObject; - const sharedMap3 = await dataStore3.getSharedObject(sharedMapId); - - await loaderContainerTracker.ensureSynchronized(); - assert.strictEqual(sharedMap3.get("1"), "B", "Contents should be as required"); - assert.strictEqual( - JSON.stringify(sharedMap3.summarize()), - JSON.stringify(sharedMapFromRC.summarize()), - "Summaries of shared string should match and contents should be same!!"); - }); + it("Container rehydration with not bounded dataStore handle stored in root of other bounded dataStore", + async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); - it("Rehydrate container, create but don't load a data store. Attach rehydrated container and load " + - "container 2 from another loader. Then load the created dataStore from container 2, make changes to dds " + - "in it check reflection of changes in rehydrated container", - async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - - // Create another dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - peerDataStore.peerDataStoreRuntimeChannel.bindToContext(); - const sharedMap1 = await dataStore2.getSharedObject(sharedMapId); - sharedMap1.set("0", "A"); - const snapshotTree = container.serialize(); - container.close(); // close the container that we don't use any more, so it doesn't block ensureSynchronized() - - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - await rehydratedContainer.attach(request); - - // Now load the container from another loader. - const urlResolver2 = driver.createUrlResolver(); - const loader2 = createTestLoader(); - assert(rehydratedContainer.resolvedUrl); - const requestUrl2 = await urlResolver2.getAbsoluteUrl(rehydratedContainer.resolvedUrl, ""); - const container2 = await loader2.resolve({ url: requestUrl2 }); - - // Get the sharedString1 from dataStore2 in container2. - const responseBefore = await container2.request({ url: `/${dataStore2.context.id}` }); - const dataStore3 = responseBefore.value as TestFluidObject; - const sharedMap3 = await dataStore3.getSharedObject(sharedMapId); - sharedMap3.set("1", "B"); - - // Get the sharedString1 from dataStore2 in rehydrated container. - const responseAfter = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); - const dataStore2FromRC = responseAfter.value as TestFluidObject; - const sharedMapFromRC = await dataStore2FromRC.getSharedObject(sharedMapId); - - await loaderContainerTracker.ensureSynchronized(); - assert.strictEqual(sharedMapFromRC.get("1"), "B", "Changes should be reflected in other map"); - assert.strictEqual( - JSON.stringify(sharedMap3.summarize()), - JSON.stringify(sharedMapFromRC.summarize()), - "Summaries of shared string should match and contents should be same!!"); - }); + // Create another dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - it("Container rehydration with not bounded dataStore handle stored in root of other bounded dataStore", - async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); + const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); + rootOfDataStore1.set("dataStore2", dataStore2.handle); - // Create another dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; + const snapshotTree = container.serialize(); + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); - rootOfDataStore1.set("dataStore2", dataStore2.handle); + const response = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); + const dataStore2FromRC = response.value as TestFluidObject; + assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); + assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); + }); - const snapshotTree = container.serialize(); - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + it("Container rehydration with not bounded dds handle stored in root of bounded dataStore", async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); - const response = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); - const dataStore2FromRC = response.value as TestFluidObject; - assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); - assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); - }); + // Create another not bounded dds + const ddsId = "notbounddds"; + const dds2 = defaultDataStore.runtime.createChannel(ddsId, SharedString.getFactory().type); - it("Container rehydration with not bounded dds handle stored in root of bounded dataStore", async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); + const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); + rootOfDataStore1.set("dd2", dds2.handle); - // Create another not bounded dds - const ddsId = "notbounddds"; - const dds2 = defaultDataStore.runtime.createChannel(ddsId, SharedString.getFactory().type); + const snapshotTree = container.serialize(); + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); - rootOfDataStore1.set("dd2", dds2.handle); + const response = await rehydratedContainer.request({ url: `/${defaultDataStore.runtime.id}/${ddsId}` }); + const ddd2FromRC = response.value as SharedString; + assert(ddd2FromRC, "ddd2 should have been serialized properly"); + assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); + assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); + }); - const snapshotTree = container.serialize(); - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + it("Container rehydration with not bounded dds handle stored in root of bound dataStore. The not bounded dds " + + "also stores handle not bounded data store", + async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); + + // Create another not bounded dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; + + // Create another not bounded dds + const ddsId = "notbounddds"; + const dds2 = defaultDataStore.runtime.createChannel(ddsId, SharedMap.getFactory().type) as SharedMap; + dds2.set("dataStore2", dataStore2.handle); + + const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); + rootOfDataStore1.set("dd2", dds2.handle); + + const snapshotTree = container.serialize(); + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + + const responseForDDS = + await rehydratedContainer.request({ url: `/${defaultDataStore.runtime.id}/${ddsId}` }); + const ddd2FromRC = responseForDDS.value as SharedString; + assert(ddd2FromRC, "ddd2 should have been serialized properly"); + assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); + assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); + + const responseForDataStore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); + const dataStore2FromRC = responseForDataStore.value as TestFluidObject; + assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); + assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); + }); - const response = await rehydratedContainer.request({ url: `/${defaultDataStore.runtime.id}/${ddsId}` }); - const ddd2FromRC = response.value as SharedString; - assert(ddd2FromRC, "ddd2 should have been serialized properly"); - assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); - assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); - }); + it("Container rehydration with not bounded data store handle stored in root of bound dataStore. " + + "The not bounded data store also stores handle not bounded dds", + async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); + + // Create another not bounded dataStore + const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); + const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; + + // Create another not bounded dds + const ddsId = "notbounddds"; + const dds2 = dataStore2.runtime.createChannel(ddsId, SharedMap.getFactory().type) as SharedMap; + const rootOfDataStore2 = await dataStore2.getSharedObject(sharedMapId); + rootOfDataStore2.set("dds2", dds2.handle); + + const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); + rootOfDataStore1.set("dataStore2", dataStore2.handle); + + const snapshotTree = container.serialize(); + const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + + const responseForDDS = await rehydratedContainer.request({ url: `/${dataStore2.runtime.id}/${ddsId}` }); + const ddd2FromRC = responseForDDS.value as SharedString; + assert(ddd2FromRC, "ddd2 should have been serialized properly"); + assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); + assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); + + const responseForDataStore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); + const dataStore2FromRC = responseForDataStore.value as TestFluidObject; + assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); + assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); + }); - it("Container rehydration with not bounded dds handle stored in root of bound dataStore. The not bounded dds " + - "also stores handle not bounded data store", - async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - - // Create another not bounded dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - - // Create another not bounded dds - const ddsId = "notbounddds"; - const dds2 = defaultDataStore.runtime.createChannel(ddsId, SharedMap.getFactory().type) as SharedMap; - dds2.set("dataStore2", dataStore2.handle); - - const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); - rootOfDataStore1.set("dd2", dds2.handle); - - const snapshotTree = container.serialize(); - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - - const responseForDDS = - await rehydratedContainer.request({ url: `/${defaultDataStore.runtime.id}/${ddsId}` }); - const ddd2FromRC = responseForDDS.value as SharedString; - assert(ddd2FromRC, "ddd2 should have been serialized properly"); - assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); - assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); - - const responseForDataStore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); - const dataStore2FromRC = responseForDataStore.value as TestFluidObject; - assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); - assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); - }); + it("Not bounded/Unreferenced data store should not get serialized on container serialization", async () => { + const { container, defaultDataStore } = + await createDetachedContainerAndGetRootDataStore(); - it("Container rehydration with not bounded data store handle stored in root of bound dataStore. The not bounded" + - "data store also stores handle not bounded dds", - async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); - - // Create another not bounded dataStore - const peerDataStore = await createPeerDataStore(defaultDataStore.context.containerRuntime); - const dataStore2 = peerDataStore.peerDataStore as TestFluidObject; - - // Create another not bounded dds - const ddsId = "notbounddds"; - const dds2 = dataStore2.runtime.createChannel(ddsId, SharedMap.getFactory().type) as SharedMap; - const rootOfDataStore2 = await dataStore2.getSharedObject(sharedMapId); - rootOfDataStore2.set("dds2", dds2.handle); - - const rootOfDataStore1 = await defaultDataStore.getSharedObject(sharedMapId); - rootOfDataStore1.set("dataStore2", dataStore2.handle); - - const snapshotTree = container.serialize(); - const rehydratedContainer = await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - - const responseForDDS = await rehydratedContainer.request({ url: `/${dataStore2.runtime.id}/${ddsId}` }); - const ddd2FromRC = responseForDDS.value as SharedString; - assert(ddd2FromRC, "ddd2 should have been serialized properly"); - assert.strictEqual(ddd2FromRC.id, ddsId, "DDS id should match"); - assert.strictEqual(ddd2FromRC.id, dds2.id, "Both dds id should match"); - - const responseForDataStore = await rehydratedContainer.request({ url: `/${dataStore2.context.id}` }); - const dataStore2FromRC = responseForDataStore.value as TestFluidObject; - assert(dataStore2FromRC, "DataStore2 should have been serialized properly"); - assert.strictEqual(dataStore2FromRC.runtime.id, dataStore2.runtime.id, "DataStore2 id should match"); - }); + // Create another not bounded dataStore + await createPeerDataStore(defaultDataStore.context.containerRuntime); - it("Not bounded/Unreferenced data store should not get serialized on container serialization", async () => { - const { container, defaultDataStore } = - await createDetachedContainerAndGetRootDataStore(); + const snapshotTree = JSON.parse(container.serialize()); - // Create another not bounded dataStore - await createPeerDataStore(defaultDataStore.context.containerRuntime); + assertProtocolTree(snapshotTree); + const { channelsTree } = assertSchedulerTree(snapshotTree); + assertDatastoreTree(snapshotTree, "default"); + assert.ok( + // eslint-disable-next-line unicorn/no-unsafe-regex + !Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), + "unbounded/unreferenced data store tree present", + ); + }); + }; - const snapshotTree = JSON.parse(container.serialize()); + // Run once with isolated channels + tests(); - assertProtocolTree(snapshotTree); - const { channelsTree } = assertSchedulerTree(snapshotTree); - assertDatastoreTree(snapshotTree, "default"); - assert.ok( - // eslint-disable-next-line unicorn/no-unsafe-regex - !Object.keys(channelsTree.trees).some((key) => /^(?:\w+-){4}\w+$/.test(key)), - "unbounded/unreferenced data store tree present", - ); - }); + // Run again with isolated channels disabled + disableIsolatedChannels = true; + describe("With isolated channels disabled", tests); }); diff --git a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts index e0b5edfdc30b..d1c762f832ff 100644 --- a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts @@ -27,12 +27,13 @@ class TestDataObject extends DataObject { public readonly getContext = () => this.context; } -async function createContainer(): Promise<{ +async function createContainer(options: { + documentId: string; + runtimeOptions: Omit; +}): Promise<{ container: IContainer; opProcessingController: OpProcessingController; }> { - const documentId = "summarizerTest"; - const codeDetails: IFluidCodeDetails = { package: "summarizerTestPackage", }; @@ -44,8 +45,9 @@ async function createContainer(): Promise<{ SharedObjectSequence.getFactory(), ], []); - const runtimeOptions: IContainerRuntimeOptions = { - generateSummaries: false, + const thisRuntimeOptions: IContainerRuntimeOptions = { + ...{ generateSummaries: false }, + ...options.runtimeOptions, }; const runtimeFactory = new ContainerRuntimeFactoryWithDefaultDataStore( @@ -56,7 +58,7 @@ async function createContainer(): Promise<{ ], undefined, undefined, - runtimeOptions, + thisRuntimeOptions, ); const deltaConnectionServer = LocalDeltaConnectionServer.create(); @@ -67,7 +69,7 @@ async function createContainer(): Promise<{ const container = await createAndAttachContainer( codeDetails, loader, - urlResolver.createCreateNewRequest(documentId)); + urlResolver.createCreateNewRequest(options.documentId)); const opProcessingController = new OpProcessingController(); opProcessingController.addDeltaManagers(container.deltaManager); @@ -77,7 +79,10 @@ async function createContainer(): Promise<{ describe("Summaries", () => { it("Should generate summary tree", async () => { - const { container, opProcessingController } = await createContainer(); + const { container, opProcessingController } = await createContainer({ + documentId: "summaryTest", + runtimeOptions: { disableIsolatedChannels: false }, + }); const defaultDataStore = await requestFluidObject(container, defaultDataStoreId); const containerRuntime = defaultDataStore.getContext().containerRuntime as ContainerRuntime; @@ -93,9 +98,9 @@ describe("Summaries", () => { // Validate stats assert(stats.handleNodeCount === 0, "Expecting no handles for first summary."); // .metadata, .component, and .attributes blobs - assert(stats.blobNodeCount >= 3, `Stats expected at least 2 blob nodes, but had ${stats.blobNodeCount}.`); + assert(stats.blobNodeCount >= 3, `Stats expected at least 3 blob nodes, but had ${stats.blobNodeCount}.`); // root node, data store .channels, default data store, dds .channels, and default root dds - assert(stats.treeNodeCount >= 5, `Stats expected at least 3 tree nodes, but had ${stats.treeNodeCount}.`); + assert(stats.treeNodeCount >= 5, `Stats expected at least 5 tree nodes, but had ${stats.treeNodeCount}.`); // Validate summary assert(!summary.unreferenced, "Root summary should be referenced."); @@ -125,4 +130,56 @@ describe("Summaries", () => { assert(gcNodeIds.includes("/default"), "Expected default data store gc node."); assert(gcNodeIds.includes("/default/root"), "Expected default root DDS gc node."); }); + + it("Should generate summary tree with isolated channels disabled", async () => { + const { container, opProcessingController } = await createContainer({ + documentId: "summaryTestWithoutIsolatedChannels", + runtimeOptions: { disableIsolatedChannels: true }, + }); + const defaultDataStore = await requestFluidObject(container, defaultDataStoreId); + const containerRuntime = defaultDataStore.getContext().containerRuntime as ContainerRuntime; + + await opProcessingController.process(); + + const { gcData, stats, summary } = await containerRuntime.summarize({ + runGC: false, + fullTree: false, + trackState: false, + summaryLogger: new TelemetryNullLogger(), + }); + + // Validate stats + assert(stats.handleNodeCount === 0, "Expecting no handles for first summary."); + // .component, and .attributes blobs + assert(stats.blobNodeCount >= 2, `Stats expected at least 2 blob nodes, but had ${stats.blobNodeCount}.`); + // root node, default data store, and default root dds + assert(stats.treeNodeCount >= 3, `Stats expected at least 3 tree nodes, but had ${stats.treeNodeCount}.`); + + // Validate summary + assert(!summary.unreferenced, "Root summary should be referenced."); + + assert(summary.tree[".metadata"] === undefined, "Unexpected .metadata blob in summary root."); + + assert(summary.tree[channelsTreeName] === undefined, "Unexpected .channels tree in summary root."); + + const defaultDataStoreNode = summary.tree[defaultDataStoreId]; + assert(defaultDataStoreNode?.type === SummaryType.Tree, "Expected default data store tree in summary."); + assert(!defaultDataStoreNode.unreferenced, "Default data store should be referenced."); + assert(defaultDataStoreNode.tree[".component"]?.type === SummaryType.Blob, + "Expected .component blob in default data store summary tree."); + assert(defaultDataStoreNode.tree[channelsTreeName] === undefined, + "Unexpected .channels tree in default data store."); + + const defaultDdsNode = defaultDataStoreNode.tree.root; + assert(defaultDdsNode?.type === SummaryType.Tree, "Expected default root DDS in summary."); + assert(!defaultDdsNode.unreferenced, "Default root DDS should be referenced."); + assert(defaultDdsNode.tree[".attributes"]?.type === SummaryType.Blob, + "Expected .attributes blob in default root DDS summary tree."); + + // Validate GC nodes + const gcNodeIds = Object.keys(gcData.gcNodes); + assert(gcNodeIds.includes("/"), "Expected root gc node."); + assert(gcNodeIds.includes("/default"), "Expected default data store gc node."); + assert(gcNodeIds.includes("/default/root"), "Expected default root DDS gc node."); + }); }); diff --git a/packages/test/test-utils/src/localCodeLoader.ts b/packages/test/test-utils/src/localCodeLoader.ts index 9d8e5cf344fa..b9613507e96f 100644 --- a/packages/test/test-utils/src/localCodeLoader.ts +++ b/packages/test/test-utils/src/localCodeLoader.ts @@ -13,6 +13,7 @@ import { import { IFluidCodeDetails, IProvideFluidCodeDetailsComparer } from "@fluidframework/core-interfaces"; import { IProvideFluidDataStoreFactory, IProvideFluidDataStoreRegistry } from "@fluidframework/runtime-definitions"; import { createDataStoreFactory } from "@fluidframework/runtime-utils"; +import { IContainerRuntimeOptions } from "@fluidframework/container-runtime"; export type SupportedExportInterfaces = Partial< IProvideRuntimeFactory & @@ -30,7 +31,10 @@ export type fluidEntryPoint = SupportedExportInterfaces | IFluidModule; export class LocalCodeLoader implements ICodeLoader { private readonly fluidPackageCache = new Map(); - constructor(packageEntries: Iterable<[IFluidCodeDetails, fluidEntryPoint]>) { + constructor( + packageEntries: Iterable<[IFluidCodeDetails, fluidEntryPoint]>, + runtimeOptions?: IContainerRuntimeOptions, + ) { for (const entry of packageEntries) { // Store the entry point against a unique id in the fluidPackageCache. // For code details containing a package name, use the package name as the id. @@ -58,7 +62,11 @@ export class LocalCodeLoader implements ICodeLoader { IRuntimeFactory: new ContainerRuntimeFactoryWithDefaultDataStore( defaultFactory, - [[defaultFactory.type, Promise.resolve(defaultFactory)]]), + [[defaultFactory.type, Promise.resolve(defaultFactory)]], + undefined, + undefined, + runtimeOptions, + ), }, }; } From c7b50ba133fed0bb7b949ed91670f13c7a74fa00 Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 9 Mar 2021 04:12:34 -0500 Subject: [PATCH 14/35] Add back missing skip --- .../src/test/detachedContainerTests.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts index 6321d49b2b72..2410fab78632 100644 --- a/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/detachedContainerTests.spec.ts @@ -576,7 +576,7 @@ const tests = (argsFactory: () => ITestObjectProvider) => { await defPromise.promise; }); - it("Fire ops during container attach for shared matrix", async () => { + it.skip("Fire ops during container attach for shared matrix", async () => { const op = { pos1: 0, seg: 9, type: 0, target: "rows" }; const defPromise = new Deferred(); const container = await loader.createDetachedContainer(args.defaultCodeDetails); From be7216bcd0f13be1a2029286b4f2dc0cb87efea6 Mon Sep 17 00:00:00 2001 From: Arin Date: Tue, 9 Mar 2021 04:52:49 -0500 Subject: [PATCH 15/35] Disable by default --- packages/runtime/container-runtime/src/containerRuntime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 86a8e61adcb0..126b569e23d9 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -688,7 +688,7 @@ export class ContainerRuntime extends TypedEventEmitter super(); this.runtimeOptions = { - ...{ disableIsolatedChannels: false }, + ...{ disableIsolatedChannels: true }, ...runtimeOptions, }; From 8bc2977b5e3796dab694da97e8f9364dedfdd1af Mon Sep 17 00:00:00 2001 From: Arin Date: Wed, 10 Mar 2021 06:15:11 -0500 Subject: [PATCH 16/35] UseContainerRuntime for checking if disabled on write; use helper functions when reading --- .../container-runtime/src/containerRuntime.ts | 36 ++++++--- .../container-runtime/src/dataStoreContext.ts | 74 ++++++++----------- .../container-runtime/src/dataStores.ts | 50 ++++++------- .../container-runtime/src/summaryFormat.ts | 15 +++- .../src/test/dataStoreContext.spec.ts | 35 ++------- .../src/test/dataStoreCreation.spec.ts | 34 ++------- .../src/test/dataStores.spec.ts | 42 ++++++++--- .../runtime-definitions/src/summary.ts | 12 ++- .../src/summarizerNode/summarizerNode.ts | 4 +- 9 files changed, 151 insertions(+), 151 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 24500958ab0e..4aee4caa24d7 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -122,7 +122,7 @@ import { SummaryCollection } from "./summaryCollection"; import { PendingStateManager } from "./pendingStateManager"; import { pkgVersion } from "./packageVersion"; import { BlobManager } from "./blobManager"; -import { DataStores, getSnapshotForDataStores } from "./dataStores"; +import { DataStores, getSummaryForDatastores } from "./dataStores"; import { blobsTreeName, chunksBlobName, @@ -611,6 +611,14 @@ export class ContainerRuntime extends TypedEventEmitter return this.context.attachState; } + /** + * Returns true if generating summaries with isolated channels is + * explicitly disabled. This only affects how summaries are written. + */ + public get disableIsolatedChannels(): boolean { + return !!this.runtimeOptions.disableIsolatedChannels; + } + public nextSummarizerP?: Promise; public nextSummarizerD?: Deferred; @@ -676,7 +684,7 @@ export class ContainerRuntime extends TypedEventEmitter private constructor( private readonly context: IContainerContext, private readonly registry: IFluidDataStoreRegistry, - metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: undefined }, + metadata: IContainerRuntimeMetadata = { summaryFormatVersion: undefined }, chunks: [string, string[]][], runtimeOptions: IContainerRuntimeOptions = { generateSummaries: true, @@ -732,7 +740,7 @@ export class ContainerRuntime extends TypedEventEmitter } this.dataStores = new DataStores( - getSnapshotForDataStores(context.baseSnapshot, metadata.snapshotFormatVersion), + getSummaryForDatastores(context.baseSnapshot, metadata), this, (attachMsg) => this.submit(ContainerMessageType.Attach, attachMsg), (id: string, createParam: CreateChildSummarizerNodeParam) => ( @@ -747,7 +755,6 @@ export class ContainerRuntime extends TypedEventEmitter getGCDataFn, getInitialGCSummaryDetailsFn, ), - !!this.runtimeOptions.disableIsolatedChannels, this._logger); this.blobManager = new BlobManager( @@ -952,6 +959,13 @@ export class ContainerRuntime extends TypedEventEmitter } } + private formMetadata(): IContainerRuntimeMetadata { + return { + summaryFormatVersion: 1, + disableIsolatedChannels: this.disableIsolatedChannels || undefined, + }; + } + /** * Retrieves the runtime for a data store if it's referenced as per the initially summary that it is loaded with. * This is a workaround to handle scenarios where a data store shared with an external app is deleted and marked @@ -981,7 +995,7 @@ export class ContainerRuntime extends TypedEventEmitter public async snapshot(): Promise { const root: ITree = { entries: [] }; - if (this.runtimeOptions.disableIsolatedChannels) { + if (this.disableIsolatedChannels) { root.entries = root.entries.concat(await this.dataStores.snapshot()); } else { root.entries.push(new TreeTreeEntry( @@ -989,7 +1003,7 @@ export class ContainerRuntime extends TypedEventEmitter { entries: await this.dataStores.snapshot() }, )); - const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; + const metadata = this.formMetadata(); root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); } @@ -1001,10 +1015,8 @@ export class ContainerRuntime extends TypedEventEmitter } private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) { - if (!this.runtimeOptions.disableIsolatedChannels) { - const metadata: IContainerRuntimeMetadata = { snapshotFormatVersion: 1 }; - addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); - } + const metadata = this.formMetadata(); + addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); if (this.chunkMap.size > 0) { const content = JSON.stringify([...this.chunkMap]); addBlobToSummary(summaryTree, chunksBlobName, content); @@ -1384,7 +1396,7 @@ export class ContainerRuntime extends TypedEventEmitter public createSummary(): ISummaryTree { const summarizeResult = this.dataStores.createSummary(); - if (!this.runtimeOptions.disableIsolatedChannels) { + if (!this.disableIsolatedChannels) { // Wrap data store summaries in .channels subtree. wrapSummaryInChannelsTree(summarizeResult); } @@ -1437,7 +1449,7 @@ export class ContainerRuntime extends TypedEventEmitter const summarizeResult = await this.dataStores.summarize(fullTree, trackState); let pathPartsForChildren: string[] | undefined; - if (!this.runtimeOptions.disableIsolatedChannels) { + if (!this.disableIsolatedChannels) { // Wrap data store summaries in .channels subtree. wrapSummaryInChannelsTree(summarizeResult); pathPartsForChildren = [channelsTreeName]; diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 7cbfb8402c92..930aaab8bc9d 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -24,7 +24,6 @@ import { Deferred, LazyPromise, TypedEventEmitter, - unreachableCase, } from "@fluidframework/common-utils"; import { IDocumentStorageService } from "@fluidframework/driver-definitions"; import { readAndParse } from "@fluidframework/driver-utils"; @@ -65,6 +64,7 @@ import { ContainerRuntime } from "./containerRuntime"; import { dataStoreAttributesBlobName, DataStoreSummaryFormatVersion, + hasIsolatedChannels, summaryFormatVersionToNumber, wrapSummaryInChannelsTree, } from "./summaryFormat"; @@ -77,8 +77,9 @@ function createAttributes( const stringifiedPkg = JSON.stringify(pkg); return { pkg: stringifiedPkg, - snapshotFormatVersion: disableIsolatedChannels ? "0.1" : 2, + snapshotFormatVersion: 2, isRootDataStore, + disableIsolatedChannels: disableIsolatedChannels || undefined, }; } export function createAttributesBlob( @@ -104,6 +105,8 @@ export interface IFluidDataStoreAttributes { * true. This will ensure that older data stores are incorrectly collected. */ readonly isRootDataStore?: boolean; + /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ + readonly disableIsolatedChannels?: true; } interface ISnapshotDetails { @@ -200,7 +203,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter void, - protected readonly disableIsolatedChannels: boolean, protected pkg?: readonly string[], ) { super(); @@ -394,7 +396,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter { throw new Error("Already attached"); }, - disableIsolatedChannels, pkg, ); } @@ -710,29 +710,16 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext { let pkgFromSnapshot: string[]; // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. - // For snapshotFormatVersion = "0.1" or above, pkg is jsonified, otherwise it is just a string. - switch (attributes.snapshotFormatVersion) { - case undefined: { - if (attributes.pkg.startsWith("[\"") && attributes.pkg.endsWith("\"]")) { - pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; - } else { - pkgFromSnapshot = [attributes.pkg]; - } - break; - } - case 2: { - tree = tree.trees[channelsTreeName]; - // Intentional fallthrough, since package is still JSON - } - case "0.1": { + // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string. + const formatVersion = summaryFormatVersionToNumber(attributes.snapshotFormatVersion); + if (formatVersion < 1) { + if (attributes.pkg.startsWith("[\"") && attributes.pkg.endsWith("\"]")) { pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; - break; - } - default: { - unreachableCase( - attributes.snapshotFormatVersion, - `Invalid snapshot format version ${attributes.snapshotFormatVersion}`); + } else { + pkgFromSnapshot = [attributes.pkg]; } + } else { + pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; } this.pkg = pkgFromSnapshot; @@ -742,6 +729,10 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext { * roots in the document but they won't break. */ isRootDataStore = attributes.isRootDataStore ?? true; + + if (hasIsolatedChannels(attributes)) { + tree = tree.trees[channelsTreeName]; + } } return { @@ -792,8 +783,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { bindChannel: (channel: IFluidDataStoreChannel) => void, private readonly snapshotTree: ISnapshotTree | undefined, protected readonly isRootDataStore: boolean, - disableIsolatedChannels: boolean, - protected readonly summaryFormatVersion: DataStoreSummaryFormatVersion, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -809,7 +798,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { snapshotTree ? BindState.Bound : BindState.NotBound, true, bindChannel, - disableIsolatedChannels, pkg); this.attachListeners(); } @@ -832,13 +820,17 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { const summarizeResult = this.channel.getAttachSummary(); - if (!this.disableIsolatedChannels) { + if (!this._containerRuntime.disableIsolatedChannels) { // Wrap dds summaries in .channels subtree. wrapSummaryInChannelsTree(summarizeResult); } // Add data store's attributes to the summary. - const attributes = createAttributes(this.pkg, this.isRootDataStore, this.disableIsolatedChannels); + const attributes = createAttributes( + this.pkg, + this.isRootDataStore, + this._containerRuntime.disableIsolatedChannels, + ); addBlobToSummary(summarizeResult, dataStoreAttributesBlobName, JSON.stringify(attributes)); // Add GC details to the summary. @@ -863,10 +855,13 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { protected async getInitialSnapshotDetails(): Promise { assert(this.pkg !== undefined, "pkg should be available in local data store"); assert(this.isRootDataStore !== undefined, "isRootDataStore should be available in local data store"); - const versionNumber = summaryFormatVersionToNumber(this.summaryFormatVersion); + const snapshot = this._containerRuntime.disableIsolatedChannels + ? this.snapshotTree + : this.snapshotTree?.trees[channelsTreeName]; return { pkg: this.pkg, - snapshot: versionNumber < 2 ? this.snapshotTree : this.snapshotTree?.trees[channelsTreeName], + snapshot: this._containerRuntime.disableIsolatedChannels ? this.snapshotTree : + snapshot, isRootDataStore: this.isRootDataStore, }; } @@ -894,8 +889,6 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { bindChannel: (channel: IFluidDataStoreChannel) => void, snapshotTree: ISnapshotTree | undefined, isRootDataStore: boolean, - disableIsolatedChannels: boolean, - summaryFormatVersion: DataStoreSummaryFormatVersion, /** * @deprecated 0.16 Issue #1635, #3631 */ @@ -911,8 +904,6 @@ export class LocalFluidDataStoreContext extends LocalFluidDataStoreContextBase { bindChannel, snapshotTree, isRootDataStore, - disableIsolatedChannels, - summaryFormatVersion, createProps); } } @@ -937,7 +928,6 @@ export class LocalDetachedFluidDataStoreContext bindChannel: (channel: IFluidDataStoreChannel) => void, snapshotTree: ISnapshotTree | undefined, isRootDataStore: boolean, - disableIsolatedChannels: boolean, ) { super( id, @@ -949,8 +939,6 @@ export class LocalDetachedFluidDataStoreContext bindChannel, snapshotTree, isRootDataStore, - disableIsolatedChannels, - disableIsolatedChannels ? "0.1" : 2, ); this.detachedRuntimeCreation = true; } diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index a7fbe31eb17d..30c467966109 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -53,9 +53,11 @@ import { LocalDetachedFluidDataStoreContext, } from "./dataStoreContext"; import { - ContainerRuntimeSummaryFormatVersion, dataStoreAttributesBlobName, + IContainerRuntimeMetadata, nonDataStorePaths, + rootHasIsolatedChannels, + summaryFormatVersionToNumber, } from "./summaryFormat"; /** @@ -78,7 +80,6 @@ export class DataStores implements IDisposable { private readonly submitAttachFn: (attachContent: any) => void, private readonly getCreateChildSummarizerNodeFn: (id: string, createParam: CreateChildSummarizerNodeParam) => CreateChildSummarizerNodeFn, - private readonly disableIsolatedChannels: boolean, baseLogger: ITelemetryBaseLogger, private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger), ) { @@ -103,10 +104,8 @@ export class DataStores implements IDisposable { this.runtime, this.runtime.storage, this.runtime.scope, - this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary }), - this.disableIsolatedChannels); + this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary })); } else { - let pkgFromSnapshot: string[]; if (typeof value !== "object") { throw new Error("Snapshot should be there to load from!!"); } @@ -116,15 +115,12 @@ export class DataStores implements IDisposable { snapshotTree.blobs, snapshotTree.blobs[dataStoreAttributesBlobName]); // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. - // For snapshotFormatVersion = "0.1" or above, pkg is jsonified, otherwise it is just a string. + // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string. // However the feature of loading a detached container from snapshot, is added when the - // snapshotFormatVersion is at least "0.1", so we don't expect it to be anything else. - if (attributes.snapshotFormatVersion === "0.1" - || attributes.snapshotFormatVersion === 2) { - pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; - } else { - throw new Error(`Invalid snapshot format version ${attributes.snapshotFormatVersion}`); - } + // snapshotFormatVersion is at least "0.1" (1), so we don't expect it to be anything else. + const formatVersion = summaryFormatVersionToNumber(attributes.snapshotFormatVersion); + assert(formatVersion > 0, `Invalid snapshot format version ${attributes.snapshotFormatVersion}`); + const pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; dataStoreContext = new LocalFluidDataStoreContext( key, @@ -139,7 +135,6 @@ export class DataStores implements IDisposable { // stores in older documents are not garbage collected incorrectly. This may lead to additional // roots in the document but they won't break. attributes.isRootDataStore ?? true, - this.disableIsolatedChannels, attributes.snapshotFormatVersion, ); } @@ -192,10 +187,13 @@ export class DataStores implements IDisposable { type: CreateSummarizerNodeSource.FromAttach, sequenceNumber: message.sequenceNumber, snapshot: attachMessage.snapshot ?? { - entries: [createAttributesBlob(pkg, true /* isRootDataStore */, this.disableIsolatedChannels)], + entries: [createAttributesBlob( + pkg, + true /* isRootDataStore */, + this.runtime.disableIsolatedChannels, + )], }, }), - this.disableIsolatedChannels, pkg); // Resolve pending gets and store off any new ones @@ -241,7 +239,6 @@ export class DataStores implements IDisposable { (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr), undefined, isRoot, - this.disableIsolatedChannels, ); this.contexts.addUnbound(context); return context; @@ -258,7 +255,6 @@ export class DataStores implements IDisposable { (cr: IFluidDataStoreChannel) => this.bindFluidDataStore(cr), undefined, isRoot, - this.disableIsolatedChannels, props, ); this.contexts.addUnbound(context); @@ -511,29 +507,29 @@ export class DataStores implements IDisposable { } } -export function getSnapshotForDataStores( +export function getSummaryForDatastores( snapshot: ISnapshotTree | undefined, - snapshotFormatVersion: ContainerRuntimeSummaryFormatVersion, + metadata: IContainerRuntimeMetadata, ): ISnapshotTree | undefined { if (!snapshot) { return undefined; } - if (snapshotFormatVersion !== undefined) { - const dataStoresSnapshot = snapshot.trees[channelsTreeName]; - assert(!!dataStoresSnapshot, `expected ${channelsTreeName} tree in snapshot`); - return dataStoresSnapshot; + if (rootHasIsolatedChannels(metadata)) { + const datastoresSnapshot = snapshot.trees[channelsTreeName]; + assert(!!datastoresSnapshot, `expected ${channelsTreeName} tree in snapshot`); + return datastoresSnapshot; } else { // back-compat: strip out all non-datastore paths before giving to DataStores object. - const dataStoresTrees: ISnapshotTree["trees"] = {}; + const datastoresTrees: ISnapshotTree["trees"] = {}; for (const [key, value] of Object.entries(snapshot.trees)) { if (!nonDataStorePaths.includes(key)) { - dataStoresTrees[key] = value; + datastoresTrees[key] = value; } } return { ...snapshot, - trees: dataStoresTrees, + trees: datastoresTrees, }; } } diff --git a/packages/runtime/container-runtime/src/summaryFormat.ts b/packages/runtime/container-runtime/src/summaryFormat.ts index d760d04063af..ad8f2b9dc346 100644 --- a/packages/runtime/container-runtime/src/summaryFormat.ts +++ b/packages/runtime/container-runtime/src/summaryFormat.ts @@ -5,6 +5,7 @@ import { SummaryType } from "@fluidframework/protocol-definitions"; import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions"; +import { IFluidDataStoreAttributes } from "./dataStoreContext"; export type ContainerRuntimeSummaryFormatVersion = /** @@ -52,7 +53,19 @@ export const chunksBlobName = ".chunks"; export const blobsTreeName = ".blobs"; export interface IContainerRuntimeMetadata { - snapshotFormatVersion: ContainerRuntimeSummaryFormatVersion; + readonly summaryFormatVersion: ContainerRuntimeSummaryFormatVersion; + /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ + readonly disableIsolatedChannels?: true; +} + +export function rootHasIsolatedChannels(metadata: IContainerRuntimeMetadata | undefined): boolean { + const version = summaryFormatVersionToNumber(metadata?.summaryFormatVersion); + return version >= 1 && !metadata?.disableIsolatedChannels; +} + +export function hasIsolatedChannels(attributes: IFluidDataStoreAttributes | undefined): boolean { + const version = summaryFormatVersionToNumber(attributes?.snapshotFormatVersion); + return version >= 2 && !attributes?.disableIsolatedChannels; } export const protocolTreeName = ".protocol"; diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index 96e7b906146d..83cc95dbd342 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -99,9 +99,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + true /* isRootDataStore */); await localDataStoreContext.realize(); const attachMessage = localDataStoreContext.generateAttachMessage(); @@ -139,9 +137,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); await localDataStoreContext.realize() .catch((error) => { @@ -173,9 +169,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); await localDataStoreContext.realize(); @@ -212,9 +206,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + true /* isRootDataStore */); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, true, "The data store should be root."); @@ -230,9 +222,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); const isRootNode = await localDataStoreContext.isRoot(); assert.strictEqual(isRootNode, false, "The data store should not be root."); @@ -250,9 +240,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - true /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + true /* isRootDataStore */); const gcData = await localDataStoreContext.getGCData(); assert.deepStrictEqual(gcData, emptyGCData, "GC data from getGCData should be empty."); @@ -277,9 +265,7 @@ describe("Data Store Context Tests", () => { createSummarizerNodeFn, attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); // Get the summarizer node for this data store which tracks its referenced state. const dataStoreSummarizerNode = summarizerNode.getChild(dataStoreId); @@ -369,7 +355,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); const isRootNode = await remotedDataStoreContext.isRoot(); @@ -414,7 +399,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); const isRootNode = await remotedDataStoreContext.isRoot(); @@ -477,7 +461,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -526,7 +509,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -582,7 +564,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); const gcData = await remotedDataStoreContext.getGCData(); @@ -620,7 +601,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); // Update used routes before calling summarize. This is a requirement for GC. @@ -662,7 +642,6 @@ describe("Data Store Context Tests", () => { new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, - false /* disableIsolatedChannels */, ); // Get the summarizer node for this data store which tracks its referenced state. diff --git a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts index a68fe5b345b9..dd678e69f98c 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts @@ -121,9 +121,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await context.realize(); @@ -148,8 +146,6 @@ describe("Data Store Creation Tests", () => { attachCb, undefined, false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2, 2); try { @@ -174,9 +170,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextA.realize(); @@ -200,9 +194,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextB.realize(); @@ -226,9 +218,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreBId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextB.realize(); @@ -249,9 +239,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreCId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextC.realize(); @@ -275,9 +263,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextFake.realize(); @@ -301,9 +287,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextFake.realize(); @@ -327,9 +311,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - false /* disableIsolatedChannels */, - 2); + false /* isRootDataStore */); try { await contextC.realize(); diff --git a/packages/runtime/container-runtime/src/test/dataStores.spec.ts b/packages/runtime/container-runtime/src/test/dataStores.spec.ts index cc09dbd88da4..31df95b986b9 100644 --- a/packages/runtime/container-runtime/src/test/dataStores.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStores.spec.ts @@ -6,12 +6,19 @@ import { strict as assert } from "assert"; import { ISnapshotTree } from "@fluidframework/protocol-definitions"; import { channelsTreeName } from "@fluidframework/runtime-definitions"; -import { getSnapshotForDataStores } from "../dataStores"; -import { nonDataStorePaths } from "../summaryFormat"; +import { getSummaryForDatastores } from "../dataStores"; +import { IContainerRuntimeMetadata, nonDataStorePaths } from "../summaryFormat"; describe("Runtime", () => { describe("Container Runtime", () => { - describe("getSnapshotForDataStores", () => { + describe("getSummaryForDatastores", () => { + const missingMetadata: IContainerRuntimeMetadata = { summaryFormatVersion: undefined }; + const version1Metadata: IContainerRuntimeMetadata = { summaryFormatVersion: 1 }; + const disabledMetadata: IContainerRuntimeMetadata = { + summaryFormatVersion: 1, + disableIsolatedChannels: true, + }; + const emptyTree = (id: string): ISnapshotTree => ({ id, blobs: {}, @@ -42,18 +49,35 @@ describe("Runtime", () => { }; it("Should return undefined for undefined snapshots", () => { - let snapshot = getSnapshotForDataStores(undefined, undefined); + let snapshot = getSummaryForDatastores(undefined, missingMetadata); + assert(snapshot === undefined); + snapshot = getSummaryForDatastores(undefined, version1Metadata); + assert(snapshot === undefined); + snapshot = getSummaryForDatastores(undefined, disabledMetadata); assert(snapshot === undefined); - snapshot = getSnapshotForDataStores(undefined, 1); + snapshot = getSummaryForDatastores(null as any, missingMetadata); assert(snapshot === undefined); - snapshot = getSnapshotForDataStores(null as any, undefined); + snapshot = getSummaryForDatastores(null as any, version1Metadata); assert(snapshot === undefined); - snapshot = getSnapshotForDataStores(null as any, 1); + snapshot = getSummaryForDatastores(null as any, disabledMetadata); assert(snapshot === undefined); }); it("Should strip out non-datastore paths for versions < 1", () => { - const snapshot = getSnapshotForDataStores(testSnapshot, undefined); + const snapshot = getSummaryForDatastores(testSnapshot, missingMetadata); + assert(snapshot, "Snapshot should be defined"); + assert.strictEqual(snapshot.id, "root-id", "Should be top-level"); + assert.strictEqual(Object.keys(snapshot.trees).length, 3, "Should have 3 datastores"); + assert.strictEqual(snapshot.trees[channelsTreeName]?.id, "channels-id", + "Should have channels tree as datastore"); + assert.strictEqual(snapshot.trees["some-datastore"]?.id, "top-datastore-1", + "Should have top datastore 1"); + assert.strictEqual(snapshot.trees["another-datastore"]?.id, "top-datastore-2", + "Should have top datastore 2"); + }); + + it("Should strip out non-datastore paths for disabled isolated channels", () => { + const snapshot = getSummaryForDatastores(testSnapshot, disabledMetadata); assert(snapshot, "Snapshot should be defined"); assert.strictEqual(snapshot.id, "root-id", "Should be top-level"); assert.strictEqual(Object.keys(snapshot.trees).length, 3, "Should have 3 datastores"); @@ -66,7 +90,7 @@ describe("Runtime", () => { }); it("Should give channels subtree for version 1", () => { - const snapshot = getSnapshotForDataStores(testSnapshot, 1); + const snapshot = getSummaryForDatastores(testSnapshot, version1Metadata); assert(snapshot, "Snapshot should be defined"); assert.strictEqual(snapshot.id, "channels-id", "Should be lower-level"); assert.strictEqual(Object.keys(snapshot.trees).length, 4, "Should have 4 datastores"); diff --git a/packages/runtime/runtime-definitions/src/summary.ts b/packages/runtime/runtime-definitions/src/summary.ts index c544e6715fa4..b730991e9698 100644 --- a/packages/runtime/runtime-definitions/src/summary.ts +++ b/packages/runtime/runtime-definitions/src/summary.ts @@ -106,14 +106,18 @@ export interface ISummarizerNode { summarize(fullTree: boolean): Promise; /** * Checks if there are any additional path parts for children that need to - * be loaded from the base summary. + * be loaded from the base summary. Additional path parts represent parts + * of the path between this SummarizerNode and any child SummarizerNodes + * that it might have. For example: if datastore "a" contains dds "b", but the + * path is "/a/.channels/b", then the additional path part is ".channels". * @param snapshot - the base summary to parse */ loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree): void; /** - * Checks if the base snapshot was created as a failure summary. If it has - * the base summary handle + outstanding ops blob, then this will return the - * innermost base summary, and update the state by tracking the outstanding ops. + * Does all the work of loadBaseSummaryWithoutDifferential. Additionally if + * the base summary is a differential summary containing handle + outstanding ops blob, + * then this will return the innermost base summary, and update the state by + * tracking the outstanding ops. * @param snapshot - the base summary to parse * @param readAndParseBlob - function to read and parse blobs from storage * @returns the base summary to be used diff --git a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts index d1e03bf80270..c9d8cc185171 100644 --- a/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts +++ b/packages/runtime/runtime-utils/src/summarizerNode/summarizerNode.ts @@ -252,7 +252,7 @@ export class SummarizerNode implements IRootSummarizerNode { const snapshotTree = await getSnapshot(); const referenceSequenceNumber = await seqFromTree(snapshotTree, readAndParseBlob); - return this.refreshLatestSummaryFromSnapshot( + await this.refreshLatestSummaryFromSnapshot( referenceSequenceNumber, snapshotTree, undefined, @@ -363,6 +363,8 @@ export class SummarizerNode implements IRootSummarizerNode { } public loadBaseSummaryWithoutDifferential(snapshot: ISnapshotTree) { + // Check base summary to see if it has any additional path parts + // separating child SummarizerNodes. Checks for .channels subtrees. const { childrenPathPart } = parseSummaryForSubtrees(snapshot); if (childrenPathPart !== undefined && this.latestSummary !== undefined) { this.latestSummary.additionalPath = EscapedPath.create(childrenPathPart); From 2a7a344dd12c5d652c805d726ba5c81bbdbdc40a Mon Sep 17 00:00:00 2001 From: Arin Date: Wed, 10 Mar 2021 18:11:33 -0500 Subject: [PATCH 17/35] Fix tests --- .../runtime/container-runtime/src/dataStoreContext.ts | 1 - packages/tools/replay-tool/src/helpers.ts | 2 +- packages/utils/tool-utils/src/snapshotNormalizer.ts | 9 +++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 930aaab8bc9d..6cc6bbebc822 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -860,7 +860,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext { : this.snapshotTree?.trees[channelsTreeName]; return { pkg: this.pkg, - snapshot: this._containerRuntime.disableIsolatedChannels ? this.snapshotTree : snapshot, isRootDataStore: this.isRootDataStore, }; diff --git a/packages/tools/replay-tool/src/helpers.ts b/packages/tools/replay-tool/src/helpers.ts index 460f33dacdb2..411cb10bfec1 100644 --- a/packages/tools/replay-tool/src/helpers.ts +++ b/packages/tools/replay-tool/src/helpers.ts @@ -66,7 +66,7 @@ export function compareWithReferenceSnapshot( const packageVersionRegex = /\\"packageversion\\":\\"[^"]+\\"/gi; const packageVersionPlaceholder = "\\\"packageVersion\\\":\\\"X\\\""; - // The snapshotFormatVersion could vary. Replace all with -1 like packageVersion. + // The summaryFormatVersion and snapshotFormatVersion could vary. Replace all with -1 like packageVersion. const snapshotFormatVersionRegex = /\\"snapshotformatversion\\":(\\"[^"]+\\"|\d+)/gi; const snapshotFormatVersionPlaceholder = "\\\"snapshotFormatVersion\\\":-1"; diff --git a/packages/utils/tool-utils/src/snapshotNormalizer.ts b/packages/utils/tool-utils/src/snapshotNormalizer.ts index 3ef8dcbcb41f..79ae5159235d 100644 --- a/packages/utils/tool-utils/src/snapshotNormalizer.ts +++ b/packages/utils/tool-utils/src/snapshotNormalizer.ts @@ -105,6 +105,7 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali // Merge blobs to normalize in the config with runtime blobs to normalize. The contents of these blobs will be // parsed and deep sorted. const blobsToNormalize = [ ...runtimeBlobsToNormalize, ...config?.blobsToNormalize ?? [] ]; + const attributesToIgnore = ["disableIsolatedChannels"]; const normalizedEntries: ITreeEntry[] = []; for (const entry of snapshot.entries) { @@ -119,6 +120,14 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali if (blobsToNormalize.includes(entry.path)) { contents = getSortedBlobContent(contents); } + if (entry.path === ".component") { + // Ignore the following properties from datastore attributes. + const attributes: Record = JSON.parse(contents); + for (const attributeToIgnore of attributesToIgnore) { + attributes[attributeToIgnore] = undefined; + } + contents = JSON.stringify(attributes); + } normalizedEntries.push(new BlobTreeEntry(entry.path, contents)); break; } From d0daa5a9fa8b47029077d05286aecfb775bd1745 Mon Sep 17 00:00:00 2001 From: Arin Date: Wed, 10 Mar 2021 19:06:52 -0500 Subject: [PATCH 18/35] Fix summaries test --- packages/test/test-end-to-end-tests/src/test/summaries.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts index 5d2f9b27e1ea..53d22bcd606b 100644 --- a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts @@ -155,7 +155,7 @@ describe("Summaries", () => { // Validate summary assert(!summary.unreferenced, "Root summary should be referenced."); - assert(summary.tree[".metadata"] === undefined, "Unexpected .metadata blob in summary root."); + assert(summary.tree[".metadata"]?.type === SummaryType.Blob, "Expected .metadata blob in summary root."); assert(summary.tree[channelsTreeName] === undefined, "Unexpected .channels tree in summary root."); From a35dc84d3e44e4d9a2a27430df3f4e7c14248c20 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 11 Mar 2021 14:07:47 -0500 Subject: [PATCH 19/35] Snapshot should always include metadata blob --- packages/runtime/container-runtime/src/containerRuntime.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 4aee4caa24d7..df17284e4994 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -1002,11 +1002,11 @@ export class ContainerRuntime extends TypedEventEmitter channelsTreeName, { entries: await this.dataStores.snapshot() }, )); - - const metadata = this.formMetadata(); - root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); } + const metadata = this.formMetadata(); + root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); + if (this.chunkMap.size > 0) { root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap]))); } From 50adf47a1474ea84f36361fa949a9aa08a4472c8 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 11 Mar 2021 14:08:01 -0500 Subject: [PATCH 20/35] Random lerna fix --- lerna-package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna-package-lock.json b/lerna-package-lock.json index ab63f7b2a88e..58c6f0f2eda2 100644 --- a/lerna-package-lock.json +++ b/lerna-package-lock.json @@ -37740,7 +37740,7 @@ "tinylicious": { "version": "0.4.18879", "resolved": "https://registry.npmjs.org/tinylicious/-/tinylicious-0.4.18879.tgz", - "integrity": "sha512-1A3q7ThOCGxxcrNOWrn0E5DF32/T6VeJK4zE11oiOgK0kmBFKkmPUV+anlcKdppBwAtdqBaVXgKwHgV3wyZz/Q==", + "integrity": "sha512-mfxPaKNEw4lme8WMBlC+bC+MQMKHD6fRtDVB1gyirlz9hW/po+oJit2r7T+ze7PNN7VDfLGYGkYBAVNeBc3S0g==", "dev": true, "requires": { "@fluidframework/common-utils": "^0.26.0", From 789273a37f9e908ccc1cad7572cfe81607a76a8a Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 11 Mar 2021 14:23:54 -0500 Subject: [PATCH 21/35] Do not modify snapshots for comparison --- packages/tools/replay-tool/src/helpers.ts | 14 ++------------ .../utils/tool-utils/src/snapshotNormalizer.ts | 13 ------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/packages/tools/replay-tool/src/helpers.ts b/packages/tools/replay-tool/src/helpers.ts index 411cb10bfec1..931750c8db68 100644 --- a/packages/tools/replay-tool/src/helpers.ts +++ b/packages/tools/replay-tool/src/helpers.ts @@ -66,20 +66,10 @@ export function compareWithReferenceSnapshot( const packageVersionRegex = /\\"packageversion\\":\\"[^"]+\\"/gi; const packageVersionPlaceholder = "\\\"packageVersion\\\":\\\"X\\\""; - // The summaryFormatVersion and snapshotFormatVersion could vary. Replace all with -1 like packageVersion. - const snapshotFormatVersionRegex = /\\"snapshotformatversion\\":(\\"[^"]+\\"|\d+)/gi; - const snapshotFormatVersionPlaceholder = "\\\"snapshotFormatVersion\\\":-1"; - const normalizedSnapshot = JSON.parse( - JSON.stringify(snapshot, undefined, 2) - .replace(packageVersionRegex, packageVersionPlaceholder) - .replace(snapshotFormatVersionRegex, snapshotFormatVersionPlaceholder), - ); + JSON.stringify(snapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder)); const normalizedReferenceSnapshot = JSON.parse( - JSON.stringify(referenceSnapshot, undefined, 2) - .replace(packageVersionRegex, packageVersionPlaceholder) - .replace(snapshotFormatVersionRegex, snapshotFormatVersionPlaceholder), - ); + JSON.stringify(referenceSnapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder)); // Put the assert in a try catch block, so that we can report errors, if any. try { diff --git a/packages/utils/tool-utils/src/snapshotNormalizer.ts b/packages/utils/tool-utils/src/snapshotNormalizer.ts index 79ae5159235d..08069950038f 100644 --- a/packages/utils/tool-utils/src/snapshotNormalizer.ts +++ b/packages/utils/tool-utils/src/snapshotNormalizer.ts @@ -105,29 +105,16 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali // Merge blobs to normalize in the config with runtime blobs to normalize. The contents of these blobs will be // parsed and deep sorted. const blobsToNormalize = [ ...runtimeBlobsToNormalize, ...config?.blobsToNormalize ?? [] ]; - const attributesToIgnore = ["disableIsolatedChannels"]; const normalizedEntries: ITreeEntry[] = []; for (const entry of snapshot.entries) { switch (entry.type) { case TreeEntry.Blob: { - if (depth === 0 && entry.path === ".metadata") { - // Ignore the root .metadata blob - continue; - } let contents = entry.value.contents; // If this blob has to be normalized, parse and sort the blob contents first. if (blobsToNormalize.includes(entry.path)) { contents = getSortedBlobContent(contents); } - if (entry.path === ".component") { - // Ignore the following properties from datastore attributes. - const attributes: Record = JSON.parse(contents); - for (const attributeToIgnore of attributesToIgnore) { - attributes[attributeToIgnore] = undefined; - } - contents = JSON.stringify(attributes); - } normalizedEntries.push(new BlobTreeEntry(entry.path, contents)); break; } From 6dc045d1a3e3c87393d48fbb237a2999e58222aa Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 11 Mar 2021 18:05:06 -0500 Subject: [PATCH 22/35] Remove extra constructor arg --- packages/runtime/container-runtime/src/dataStores.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index 30c467966109..bb0bc9c545ae 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -135,7 +135,6 @@ export class DataStores implements IDisposable { // stores in older documents are not garbage collected incorrectly. This may lead to additional // roots in the document but they won't break. attributes.isRootDataStore ?? true, - attributes.snapshotFormatVersion, ); } this.contexts.addBoundOrRemoted(dataStoreContext); From a9f96ccc7846d4ebd2549ad07132a5bc25400cb3 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 11 Mar 2021 18:09:59 -0500 Subject: [PATCH 23/35] only expose disableIsolatedChannels instead of entire ContainerRuntime --- .../container-runtime/src/dataStoreContext.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 6cc6bbebc822..da415e2bf2fa 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -190,6 +190,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter { assert(this.pkg !== undefined, "pkg should be available in local data store"); assert(this.isRootDataStore !== undefined, "isRootDataStore should be available in local data store"); - const snapshot = this._containerRuntime.disableIsolatedChannels + const snapshot = this.disableIsolatedChannels ? this.snapshotTree : this.snapshotTree?.trees[channelsTreeName]; return { From b1a8eea30b2510af870208181499fca0f2427910 Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 12 Mar 2021 13:55:46 -0500 Subject: [PATCH 24/35] Point to updated generated snapshot test data --- packages/test/snapshots/content | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/snapshots/content b/packages/test/snapshots/content index 8229cfcffd67..7c5f9b268370 160000 --- a/packages/test/snapshots/content +++ b/packages/test/snapshots/content @@ -1 +1 @@ -Subproject commit 8229cfcffd675b59da8172b6f7e7ae8df813a9b6 +Subproject commit 7c5f9b268370329f242a451fe05cdf45050067d6 From db328c6ea22f8fdaf39e9630204389c402d66324 Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 12 Mar 2021 13:57:33 -0500 Subject: [PATCH 25/35] Inline formMetadata calls --- packages/runtime/container-runtime/src/containerRuntime.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index df17284e4994..32d0acbbd8ea 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -1004,8 +1004,7 @@ export class ContainerRuntime extends TypedEventEmitter )); } - const metadata = this.formMetadata(); - root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(metadata))); + root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata()))); if (this.chunkMap.size > 0) { root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap]))); @@ -1015,8 +1014,7 @@ export class ContainerRuntime extends TypedEventEmitter } private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) { - const metadata = this.formMetadata(); - addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata)); + addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata())); if (this.chunkMap.size > 0) { const content = JSON.stringify([...this.chunkMap]); addBlobToSummary(summaryTree, chunksBlobName, content); From b5317e8a9fcf57c4ead7c35bcf5d8df42cbab9ce Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 12 Mar 2021 15:25:09 -0500 Subject: [PATCH 26/35] Switch datastore attributes to summaryFormatVersion --- .../container-runtime/src/dataStoreContext.ts | 31 ++---- .../container-runtime/src/dataStores.ts | 8 +- .../container-runtime/src/summaryFormat.ts | 97 +++++++++++++------ .../src/test/dataStoreContext.spec.ts | 54 ++++++----- packages/test/snapshots/content | 2 +- .../src/test/contextReload.spec.ts | 4 +- packages/tools/replay-tool/src/helpers.ts | 6 +- .../tool-utils/src/snapshotNormalizer.ts | 16 +-- 8 files changed, 114 insertions(+), 104 deletions(-) diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index da415e2bf2fa..415079f2fdd4 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -63,21 +63,22 @@ import { addBlobToSummary, convertSummaryTreeToITree } from "@fluidframework/run import { ContainerRuntime } from "./containerRuntime"; import { dataStoreAttributesBlobName, - DataStoreSummaryFormatVersion, hasIsolatedChannels, - summaryFormatVersionToNumber, wrapSummaryInChannelsTree, + ReadFluidDataStoreAttributes, + WriteFluidDataStoreAttributes, + getAttributesFormatVersion, } from "./summaryFormat"; function createAttributes( pkg: readonly string[], isRootDataStore: boolean, disableIsolatedChannels: boolean, -): IFluidDataStoreAttributes { +): WriteFluidDataStoreAttributes { const stringifiedPkg = JSON.stringify(pkg); return { pkg: stringifiedPkg, - snapshotFormatVersion: 2, + summaryFormatVersion: 2, isRootDataStore, disableIsolatedChannels: disableIsolatedChannels || undefined, }; @@ -91,24 +92,6 @@ export function createAttributesBlob( return new BlobTreeEntry(dataStoreAttributesBlobName, JSON.stringify(attributes)); } -/** - * Added IFluidDataStoreAttributes similar to IChannelAttributes which will tell the attributes of a - * store like the package, snapshotFormatVersion to take different decisions based on a particular - * snapshotFormatVersion. - */ -export interface IFluidDataStoreAttributes { - pkg: string; - readonly snapshotFormatVersion: DataStoreSummaryFormatVersion; - /** - * This tells whether a data store is root. Root data stores are never collected. - * Non-root data stores may be collected if they are not used. If this is not present, default it to - * true. This will ensure that older data stores are incorrectly collected. - */ - readonly isRootDataStore?: boolean; - /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ - readonly disableIsolatedChannels?: true; -} - interface ISnapshotDetails { pkg: readonly string[]; /** @@ -710,12 +693,12 @@ export class RemotedFluidDataStoreContext extends FluidDataStoreContext { if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) { // Need to rip through snapshot and use that to populate extraBlobs const attributes = - await localReadAndParse(tree.blobs[dataStoreAttributesBlobName]); + await localReadAndParse(tree.blobs[dataStoreAttributesBlobName]); let pkgFromSnapshot: string[]; // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string. - const formatVersion = summaryFormatVersionToNumber(attributes.snapshotFormatVersion); + const formatVersion = getAttributesFormatVersion(attributes); if (formatVersion < 1) { if (attributes.pkg.startsWith("[\"") && attributes.pkg.endsWith("\"]")) { pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index bb0bc9c545ae..4b862a5cb0fb 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -47,7 +47,6 @@ import { ContainerRuntime } from "./containerRuntime"; import { FluidDataStoreContext, RemotedFluidDataStoreContext, - IFluidDataStoreAttributes, LocalFluidDataStoreContext, createAttributesBlob, LocalDetachedFluidDataStoreContext, @@ -57,7 +56,8 @@ import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels, - summaryFormatVersionToNumber, + ReadFluidDataStoreAttributes, + getAttributesFormatVersion, } from "./summaryFormat"; /** @@ -111,14 +111,14 @@ export class DataStores implements IDisposable { } const snapshotTree = value; // Need to rip through snapshot. - const attributes = readAndParseFromBlobs( + const attributes = readAndParseFromBlobs( snapshotTree.blobs, snapshotTree.blobs[dataStoreAttributesBlobName]); // Use the snapshotFormatVersion to determine how the pkg is encoded in the snapshot. // For snapshotFormatVersion = "0.1" (1) or above, pkg is jsonified, otherwise it is just a string. // However the feature of loading a detached container from snapshot, is added when the // snapshotFormatVersion is at least "0.1" (1), so we don't expect it to be anything else. - const formatVersion = summaryFormatVersionToNumber(attributes.snapshotFormatVersion); + const formatVersion = getAttributesFormatVersion(attributes); assert(formatVersion > 0, `Invalid snapshot format version ${attributes.snapshotFormatVersion}`); const pkgFromSnapshot = JSON.parse(attributes.pkg) as string[]; diff --git a/packages/runtime/container-runtime/src/summaryFormat.ts b/packages/runtime/container-runtime/src/summaryFormat.ts index ad8f2b9dc346..69843026fa24 100644 --- a/packages/runtime/container-runtime/src/summaryFormat.ts +++ b/packages/runtime/container-runtime/src/summaryFormat.ts @@ -5,7 +5,6 @@ import { SummaryType } from "@fluidframework/protocol-definitions"; import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions"; -import { IFluidDataStoreAttributes } from "./dataStoreContext"; export type ContainerRuntimeSummaryFormatVersion = /** @@ -19,38 +18,58 @@ export type ContainerRuntimeSummaryFormatVersion = */ | 1; -export type DataStoreSummaryFormatVersion = +interface IFluidDataStoreAttributes1 { + pkg: string; + readonly snapshotFormatVersion?: "0.1"; /** - * Version 0: format version is missing from summary. - * This indicates it is an older version. - */ - | undefined - /** - * Version 1: from this version the pkg within the data store - * attributes blob is a JSON array rather than a string. + * This tells whether a data store is root. Root data stores are never collected. + * Non-root data stores may be collected if they are not used. If this is not present, default it to + * true. This will ensure that older data stores are incorrectly collected. */ - | "0.1" - /** - * Introduces .channels trees for isolation of - * channel trees from data store objects. - */ - | 2; + readonly isRootDataStore?: boolean; +} +interface IFluidDataStoreAttributes2 extends IFluidDataStoreAttributes1 { + readonly snapshotFormatVersion?: undefined; + readonly summaryFormatVersion: 2; + /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ + readonly disableIsolatedChannels?: true; +} +/** + * Added IFluidDataStoreAttributes similar to IChannelAttributes which will tell the attributes of a + * store like the package, snapshotFormatVersion to take different decisions based on a particular + * snapshotFormatVersion. + */ +export type ReadFluidDataStoreAttributes = IFluidDataStoreAttributes1 | IFluidDataStoreAttributes2; +export type WriteFluidDataStoreAttributes = IFluidDataStoreAttributes2; -export function summaryFormatVersionToNumber( - version: ContainerRuntimeSummaryFormatVersion | DataStoreSummaryFormatVersion, -): number { - if (version === undefined) { - return 0; - } - if (version === "0.1") { +export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttributes): number { + if ("summaryFormatVersion" in attributes) { + /** + * Version 2+: Introduces .channels trees for isolation of + * channel trees from data store objects. + */ + return attributes.summaryFormatVersion; + } else if (attributes.snapshotFormatVersion === "0.1") { + /** + * Version 1: from this version the pkg within the data store + * attributes blob is a JSON array rather than a string. + */ return 1; } - return version; + /** + * Version 0: format version is missing from summary. + * This indicates it is an older version. + */ + return 0; } -export const metadataBlobName = ".metadata"; -export const chunksBlobName = ".chunks"; -export const blobsTreeName = ".blobs"; +export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): boolean { + const version = getAttributesFormatVersion(attributes); + if (version < 2) { + return false; + } + return !("disableIsolatedChannels" in attributes && attributes.disableIsolatedChannels); +} export interface IContainerRuntimeMetadata { readonly summaryFormatVersion: ContainerRuntimeSummaryFormatVersion; @@ -58,14 +77,28 @@ export interface IContainerRuntimeMetadata { readonly disableIsolatedChannels?: true; } -export function rootHasIsolatedChannels(metadata: IContainerRuntimeMetadata | undefined): boolean { - const version = summaryFormatVersionToNumber(metadata?.summaryFormatVersion); - return version >= 1 && !metadata?.disableIsolatedChannels; +// eslint-disable-next-line prefer-arrow/prefer-arrow-functions +export function getMetadataFormatVersion(metadata: IContainerRuntimeMetadata | undefined): number { + /** + * Version 1+: Introduces .metadata blob and .channels trees for isolation of + * data store trees from container-level objects. + * + * Version 0: metadata blob missing; format version is missing from summary. + * This indicates it is an older version. + */ + return metadata?.summaryFormatVersion ?? 0; } -export function hasIsolatedChannels(attributes: IFluidDataStoreAttributes | undefined): boolean { - const version = summaryFormatVersionToNumber(attributes?.snapshotFormatVersion); - return version >= 2 && !attributes?.disableIsolatedChannels; +export const metadataBlobName = ".metadata"; +export const chunksBlobName = ".chunks"; +export const blobsTreeName = ".blobs"; + +export function rootHasIsolatedChannels(metadata: IContainerRuntimeMetadata | undefined): boolean { + const version = getMetadataFormatVersion(metadata); + if (version < 1) { + return false; + } + return !(metadata && "disableIsolatedChannels" in metadata && metadata.disableIsolatedChannels); } export const protocolTreeName = ".protocol"; diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index 83cc95dbd342..523b19228f0b 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -29,12 +29,15 @@ import { MockFluidDataStoreRuntime } from "@fluidframework/test-runtime-utils"; import { createRootSummarizerNodeWithGC, IRootSummarizerNodeWithGC } from "@fluidframework/runtime-utils"; import { stringToBuffer, TelemetryNullLogger } from "@fluidframework/common-utils"; import { - IFluidDataStoreAttributes, LocalFluidDataStoreContext, RemotedFluidDataStoreContext, } from "../dataStoreContext"; import { ContainerRuntime } from "../containerRuntime"; -import { dataStoreAttributesBlobName } from "../summaryFormat"; +import { + dataStoreAttributesBlobName, + ReadFluidDataStoreAttributes, + WriteFluidDataStoreAttributes, +} from "../summaryFormat"; describe("Data Store Context Tests", () => { const dataStoreId = "Test1"; @@ -107,17 +110,18 @@ describe("Data Store Context Tests", () => { const attributesEntry = attachMessage.snapshot.entries.find( (e) => e.path === dataStoreAttributesBlobName); assert(attributesEntry !== undefined, "There is no attributes blob in the summary tree"); - const contents = JSON.parse((attributesEntry.value as IBlob).contents) as IFluidDataStoreAttributes; - const dataStoreAttributes: IFluidDataStoreAttributes = { + // Assume that it is in write format, will see errors if not. + const contents = JSON.parse((attributesEntry.value as IBlob).contents) as WriteFluidDataStoreAttributes; + const dataStoreAttributes: WriteFluidDataStoreAttributes = { pkg: JSON.stringify(["TestDataStore1"]), - snapshotFormatVersion: 2, + summaryFormatVersion: 2, isRootDataStore: true, }; assert.strictEqual(contents.pkg, dataStoreAttributes.pkg, "Local DataStore package does not match."); assert.strictEqual( - contents.snapshotFormatVersion, - dataStoreAttributes.snapshotFormatVersion, + contents.summaryFormatVersion, + dataStoreAttributes.summaryFormatVersion, "Local DataStore snapshot version does not match."); assert.strictEqual( contents.isRootDataStore, @@ -177,17 +181,17 @@ describe("Data Store Context Tests", () => { const attributesEntry = attachMessage.snapshot.entries.find( (e) => e.path === dataStoreAttributesBlobName); assert(attributesEntry !== undefined, "There is no attributes blob in the summary tree"); - const contents = JSON.parse((attributesEntry.value as IBlob).contents) as IFluidDataStoreAttributes; - const dataStoreAttributes: IFluidDataStoreAttributes = { + const contents = JSON.parse((attributesEntry.value as IBlob).contents) as WriteFluidDataStoreAttributes; + const dataStoreAttributes: WriteFluidDataStoreAttributes = { pkg: JSON.stringify(["TestComp", "SubComp"]), - snapshotFormatVersion: 2, + summaryFormatVersion: 2, isRootDataStore: false, }; assert.strictEqual(contents.pkg, dataStoreAttributes.pkg, "Local DataStore package does not match."); assert.strictEqual( - contents.snapshotFormatVersion, - dataStoreAttributes.snapshotFormatVersion, + contents.summaryFormatVersion, + dataStoreAttributes.summaryFormatVersion, "Local DataStore snapshot version does not match."); assert.strictEqual( contents.isRootDataStore, @@ -287,7 +291,7 @@ describe("Data Store Context Tests", () => { describe("RemoteDataStoreContext", () => { let remotedDataStoreContext: RemotedFluidDataStoreContext; - let dataStoreAttributes: IFluidDataStoreAttributes; + let dataStoreAttributes: ReadFluidDataStoreAttributes; const storage: Partial = {}; let scope: IFluidObject; let containerRuntime: ContainerRuntime; @@ -337,7 +341,7 @@ describe("Data Store Context Tests", () => { it("can correctly initialize and generate attributes", async () => { dataStoreAttributes = { pkg: JSON.stringify(["TestDataStore1"]), - snapshotFormatVersion: 2, + summaryFormatVersion: 2, isRootDataStore: true, }; const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); @@ -367,11 +371,11 @@ describe("Data Store Context Tests", () => { "summarize should always return a tree when fullTree is true"); const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; - const contents = JSON.parse(blob.content as string) as IFluidDataStoreAttributes; + const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; assert.strictEqual(contents.pkg, dataStoreAttributes.pkg, "Remote DataStore package does not match."); assert.strictEqual( - contents.snapshotFormatVersion, - dataStoreAttributes.snapshotFormatVersion, + contents.summaryFormatVersion, + dataStoreAttributes.summaryFormatVersion, "Remote DataStore snapshot version does not match."); assert.strictEqual( contents.isRootDataStore, @@ -382,7 +386,7 @@ describe("Data Store Context Tests", () => { it("can correctly initialize and generate attributes without version and isRootDataStore", async () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); @@ -409,13 +413,13 @@ describe("Data Store Context Tests", () => { "summarize should always return a tree when fullTree is true"); const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; - const contents = JSON.parse(blob.content as string) as IFluidDataStoreAttributes; + const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; assert.strictEqual( contents.pkg, JSON.stringify([dataStoreAttributes.pkg]), "Remote DataStore package does not match."); assert.strictEqual( - contents.snapshotFormatVersion, + contents.summaryFormatVersion, 2, "Remote DataStore snapshot version does not match."); // Remote context without the isRootDataStore flag in the snapshot should default it to true. @@ -442,7 +446,7 @@ describe("Data Store Context Tests", () => { it("can generate GC data without GC details in initial summary", async () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); @@ -481,7 +485,7 @@ describe("Data Store Context Tests", () => { it("can generate GC data with emtpy GC details in initial summary", async () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const gcDetails: IGarbageCollectionSummaryDetails = { usedRoutes: [], @@ -531,7 +535,7 @@ describe("Data Store Context Tests", () => { it("can generate GC data with GC details in initial summary", async () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const gcDetails: IGarbageCollectionSummaryDetails = { usedRoutes: [], @@ -573,7 +577,7 @@ describe("Data Store Context Tests", () => { it("should not reuse summary data when used state changed since last summary", async () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const gcDetails: IGarbageCollectionSummaryDetails = { usedRoutes: [""], // Set initial used routes to be same as the default used routes. @@ -624,7 +628,7 @@ describe("Data Store Context Tests", () => { it("can successfully update referenced state", () => { dataStoreAttributes = { pkg: "TestDataStore1", - snapshotFormatVersion: undefined, + summaryFormatVersion: undefined, }; const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); diff --git a/packages/test/snapshots/content b/packages/test/snapshots/content index 7c5f9b268370..8f3ed57990af 160000 --- a/packages/test/snapshots/content +++ b/packages/test/snapshots/content @@ -1 +1 @@ -Subproject commit 7c5f9b268370329f242a451fe05cdf45050067d6 +Subproject commit 8f3ed57990af852df66fd3d9b40efed1f906c72d diff --git a/packages/test/test-end-to-end-tests/src/test/contextReload.spec.ts b/packages/test/test-end-to-end-tests/src/test/contextReload.spec.ts index ef12267735c1..7fb57ea19ebd 100644 --- a/packages/test/test-end-to-end-tests/src/test/contextReload.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/contextReload.spec.ts @@ -307,7 +307,7 @@ describe("context reload (hot-swap)", function() { ); } - describe(`compat N${compatVersions} - old loader, new runtime`, () => { + describe(`compat N${compatVersion} - old loader, new runtime`, () => { beforeEach(async function() { const documentId = createDocumentId(); container = await createContainer( @@ -326,7 +326,7 @@ describe("context reload (hot-swap)", function() { tests(); }); - describe(`compat N${compatVersions} - new loader, old runtime`, () => { + describe(`compat N${compatVersion} - new loader, old runtime`, () => { beforeEach(async function() { container = await createContainer( [ diff --git a/packages/tools/replay-tool/src/helpers.ts b/packages/tools/replay-tool/src/helpers.ts index 931750c8db68..d49f008a3f52 100644 --- a/packages/tools/replay-tool/src/helpers.ts +++ b/packages/tools/replay-tool/src/helpers.ts @@ -67,9 +67,11 @@ export function compareWithReferenceSnapshot( const packageVersionPlaceholder = "\\\"packageVersion\\\":\\\"X\\\""; const normalizedSnapshot = JSON.parse( - JSON.stringify(snapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder)); + JSON.stringify(snapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder), + ); const normalizedReferenceSnapshot = JSON.parse( - JSON.stringify(referenceSnapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder)); + JSON.stringify(referenceSnapshot, undefined, 2).replace(packageVersionRegex, packageVersionPlaceholder), + ); // Put the assert in a try catch block, so that we can report errors, if any. try { diff --git a/packages/utils/tool-utils/src/snapshotNormalizer.ts b/packages/utils/tool-utils/src/snapshotNormalizer.ts index 08069950038f..3555228b0e83 100644 --- a/packages/utils/tool-utils/src/snapshotNormalizer.ts +++ b/packages/utils/tool-utils/src/snapshotNormalizer.ts @@ -92,16 +92,12 @@ function getSortedBlobContent(content: string): string { * Helper function that normalizes the given snapshot tree. It sorts objects and arrays in the snapshot. It also * normalizes certain blob contents for which the order of content does not matter. For example, garbage collection * blobs contains objects / arrays whose element order do not matter. - * We also promote any .channels subtrees, and exclude the root .metadata blob for backwards compatibility. - * In a future version, once we can reasonably expect most/all of our test criteria to have the new format with - * .channels subtrees and .metadata blob, we should invert this logic to expect .channels and .metadata, - * and add specifically for the tests (if any) that are in the older format. * @param snapshot - The snapshot tree to normalize. * @param config - Configs to use when normalizing snapshot. For example, it can contain paths of blobs whose contents * should be normalized as well. * @returns a copy of the normalized snapshot tree. */ -export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormalizerConfig, depth = 0): ITree { +export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormalizerConfig): ITree { // Merge blobs to normalize in the config with runtime blobs to normalize. The contents of these blobs will be // parsed and deep sorted. const blobsToNormalize = [ ...runtimeBlobsToNormalize, ...config?.blobsToNormalize ?? [] ]; @@ -119,15 +115,7 @@ export function getNormalizedSnapshot(snapshot: ITree, config?: ISnapshotNormali break; } case TreeEntry.Tree: { - const normalizedSubtree = getNormalizedSnapshot(entry.value, config, depth + 1); - if (entry.path === ".channels") { - // These special subtrees get promoted. - for (const subTreeEntry of normalizedSubtree.entries) { - normalizedEntries.push(subTreeEntry); - } - } else { - normalizedEntries.push(new TreeTreeEntry(entry.path, normalizedSubtree)); - } + normalizedEntries.push(new TreeTreeEntry(entry.path, getNormalizedSnapshot(entry.value, config))); break; } case TreeEntry.Attachment: { From cd6dd0214d7f32b9b645433c27c68ea1b493083f Mon Sep 17 00:00:00 2001 From: Arin Date: Fri, 12 Mar 2021 16:06:06 -0500 Subject: [PATCH 27/35] Use undefined for metadata --- .../container-runtime/src/containerRuntime.ts | 2 +- .../runtime/container-runtime/src/dataStores.ts | 2 +- .../container-runtime/src/summaryFormat.ts | 14 +------------- .../container-runtime/src/test/dataStores.spec.ts | 15 +++++++-------- 4 files changed, 10 insertions(+), 23 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 32d0acbbd8ea..3ad2f816c3fd 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -684,7 +684,7 @@ export class ContainerRuntime extends TypedEventEmitter private constructor( private readonly context: IContainerContext, private readonly registry: IFluidDataStoreRegistry, - metadata: IContainerRuntimeMetadata = { summaryFormatVersion: undefined }, + metadata: IContainerRuntimeMetadata | undefined, chunks: [string, string[]][], runtimeOptions: IContainerRuntimeOptions = { generateSummaries: true, diff --git a/packages/runtime/container-runtime/src/dataStores.ts b/packages/runtime/container-runtime/src/dataStores.ts index 4b862a5cb0fb..023c95cdf065 100644 --- a/packages/runtime/container-runtime/src/dataStores.ts +++ b/packages/runtime/container-runtime/src/dataStores.ts @@ -508,7 +508,7 @@ export class DataStores implements IDisposable { export function getSummaryForDatastores( snapshot: ISnapshotTree | undefined, - metadata: IContainerRuntimeMetadata, + metadata: IContainerRuntimeMetadata | undefined, ): ISnapshotTree | undefined { if (!snapshot) { return undefined; diff --git a/packages/runtime/container-runtime/src/summaryFormat.ts b/packages/runtime/container-runtime/src/summaryFormat.ts index 69843026fa24..ccf867b9f9f7 100644 --- a/packages/runtime/container-runtime/src/summaryFormat.ts +++ b/packages/runtime/container-runtime/src/summaryFormat.ts @@ -6,18 +6,6 @@ import { SummaryType } from "@fluidframework/protocol-definitions"; import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions"; -export type ContainerRuntimeSummaryFormatVersion = - /** - * Version 0: format version is missing from summary. - * This indicates it is an older version. - */ - | undefined - /** - * Introduces .metadata blob and .channels trees for isolation of - * data store trees from container-level objects. - */ - | 1; - interface IFluidDataStoreAttributes1 { pkg: string; readonly snapshotFormatVersion?: "0.1"; @@ -72,7 +60,7 @@ export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): b } export interface IContainerRuntimeMetadata { - readonly summaryFormatVersion: ContainerRuntimeSummaryFormatVersion; + readonly summaryFormatVersion: 1; /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ readonly disableIsolatedChannels?: true; } diff --git a/packages/runtime/container-runtime/src/test/dataStores.spec.ts b/packages/runtime/container-runtime/src/test/dataStores.spec.ts index 31df95b986b9..e5f19a62c352 100644 --- a/packages/runtime/container-runtime/src/test/dataStores.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStores.spec.ts @@ -12,8 +12,7 @@ import { IContainerRuntimeMetadata, nonDataStorePaths } from "../summaryFormat"; describe("Runtime", () => { describe("Container Runtime", () => { describe("getSummaryForDatastores", () => { - const missingMetadata: IContainerRuntimeMetadata = { summaryFormatVersion: undefined }; - const version1Metadata: IContainerRuntimeMetadata = { summaryFormatVersion: 1 }; + const enabledMetadata: IContainerRuntimeMetadata = { summaryFormatVersion: 1 }; const disabledMetadata: IContainerRuntimeMetadata = { summaryFormatVersion: 1, disableIsolatedChannels: true, @@ -49,22 +48,22 @@ describe("Runtime", () => { }; it("Should return undefined for undefined snapshots", () => { - let snapshot = getSummaryForDatastores(undefined, missingMetadata); + let snapshot = getSummaryForDatastores(undefined, undefined); assert(snapshot === undefined); - snapshot = getSummaryForDatastores(undefined, version1Metadata); + snapshot = getSummaryForDatastores(undefined, enabledMetadata); assert(snapshot === undefined); snapshot = getSummaryForDatastores(undefined, disabledMetadata); assert(snapshot === undefined); - snapshot = getSummaryForDatastores(null as any, missingMetadata); + snapshot = getSummaryForDatastores(null as any, undefined); assert(snapshot === undefined); - snapshot = getSummaryForDatastores(null as any, version1Metadata); + snapshot = getSummaryForDatastores(null as any, enabledMetadata); assert(snapshot === undefined); snapshot = getSummaryForDatastores(null as any, disabledMetadata); assert(snapshot === undefined); }); it("Should strip out non-datastore paths for versions < 1", () => { - const snapshot = getSummaryForDatastores(testSnapshot, missingMetadata); + const snapshot = getSummaryForDatastores(testSnapshot, undefined); assert(snapshot, "Snapshot should be defined"); assert.strictEqual(snapshot.id, "root-id", "Should be top-level"); assert.strictEqual(Object.keys(snapshot.trees).length, 3, "Should have 3 datastores"); @@ -90,7 +89,7 @@ describe("Runtime", () => { }); it("Should give channels subtree for version 1", () => { - const snapshot = getSummaryForDatastores(testSnapshot, version1Metadata); + const snapshot = getSummaryForDatastores(testSnapshot, enabledMetadata); assert(snapshot, "Snapshot should be defined"); assert.strictEqual(snapshot.id, "channels-id", "Should be lower-level"); assert.strictEqual(Object.keys(snapshot.trees).length, 4, "Should have 4 datastores"); From 111c6b073605dde4724fba48095d0706f5d489fe Mon Sep 17 00:00:00 2001 From: Arin Date: Wed, 17 Mar 2021 14:12:03 -0400 Subject: [PATCH 28/35] Small PR changes --- packages/runtime/container-runtime/src/summaryFormat.ts | 2 +- .../container-runtime/src/test/dataStoreCreation.spec.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/runtime/container-runtime/src/summaryFormat.ts b/packages/runtime/container-runtime/src/summaryFormat.ts index ccf867b9f9f7..63aa61cd7b81 100644 --- a/packages/runtime/container-runtime/src/summaryFormat.ts +++ b/packages/runtime/container-runtime/src/summaryFormat.ts @@ -31,7 +31,7 @@ export type ReadFluidDataStoreAttributes = IFluidDataStoreAttributes1 | IFluidDa export type WriteFluidDataStoreAttributes = IFluidDataStoreAttributes2; export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttributes): number { - if ("summaryFormatVersion" in attributes) { + if ("summaryFormatVersion" in attributes && attributes.summaryFormatVersion !== undefined) { /** * Version 2+: Introduces .channels trees for isolation of * channel trees from data store objects. diff --git a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts index dd678e69f98c..ff967dfd7195 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreCreation.spec.ts @@ -145,8 +145,7 @@ describe("Data Store Creation Tests", () => { getCreateSummarizerNodeFn(dataStoreId), attachCb, undefined, - false /* isRootDataStore */, - 2); + false /* isRootDataStore */); try { await context.realize(); From eb0fbb80bd6783490a9283254b220d037d425f7e Mon Sep 17 00:00:00 2001 From: Arin Date: Wed, 17 Mar 2021 14:55:26 -0400 Subject: [PATCH 29/35] Add a few more tests --- .../src/test/dataStoreContext.spec.ts | 76 +++++++++++++++++-- 1 file changed, 68 insertions(+), 8 deletions(-) diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index 523b19228f0b..ea996667b6df 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -384,10 +384,53 @@ describe("Data Store Context Tests", () => { }); it("can correctly initialize and generate attributes without version and isRootDataStore", async () => { + dataStoreAttributes = { pkg: "TestDataStore1" }; + const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); + const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); + const snapshotTree: ISnapshotTree = { + blobs: { [dataStoreAttributesBlobName]: "fluidDataStoreAttributes" }, + commits: {}, + trees: {}, + }; + + remotedDataStoreContext = new RemotedFluidDataStoreContext( + dataStoreId, + snapshotTree, + containerRuntime, + new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), + scope, + createSummarizerNodeFn, + ); + + const isRootNode = await remotedDataStoreContext.isRoot(); + assert.strictEqual(isRootNode, true, "The data store should be root."); + + const summarizeResult = await remotedDataStoreContext.summarize(true /* fullTree */); + assert(summarizeResult.summary.type === SummaryType.Tree, + "summarize should always return a tree when fullTree is true"); + const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; + + const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; + assert.strictEqual( + contents.pkg, + JSON.stringify([dataStoreAttributes.pkg]), + "Remote DataStore package does not match."); + assert.strictEqual( + contents.summaryFormatVersion, + 2, + "Remote DataStore snapshot version does not match."); + // Remote context without the isRootDataStore flag in the snapshot should default it to true. + assert.strictEqual(contents.isRootDataStore, true, "Remote DataStore root state does not match."); + }); + + it("can correctly initialize and generate attributes with isolated channels disabled", async () => { dataStoreAttributes = { - pkg: "TestDataStore1", - summaryFormatVersion: undefined, + pkg: JSON.stringify(["TestDataStore1"]), + summaryFormatVersion: 2, + isRootDataStore: true, }; + (containerRuntime as any).disableIsolatedChannels = true; + const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); const snapshotTree: ISnapshotTree = { @@ -416,7 +459,7 @@ describe("Data Store Context Tests", () => { const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; assert.strictEqual( contents.pkg, - JSON.stringify([dataStoreAttributes.pkg]), + dataStoreAttributes.pkg, "Remote DataStore package does not match."); assert.strictEqual( contents.summaryFormatVersion, @@ -424,6 +467,7 @@ describe("Data Store Context Tests", () => { "Remote DataStore snapshot version does not match."); // Remote context without the isRootDataStore flag in the snapshot should default it to true. assert.strictEqual(contents.isRootDataStore, true, "Remote DataStore root state does not match."); + assert.strictEqual(contents.disableIsolatedChannels, true, "Disable isolated channels should be true."); }); }); @@ -625,11 +669,7 @@ describe("Data Store Context Tests", () => { "summarize should return a tree since used state changed"); }); - it("can successfully update referenced state", () => { - dataStoreAttributes = { - pkg: "TestDataStore1", - summaryFormatVersion: undefined, - }; + function updateReferencedStateTest() { const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); const snapshotTree: ISnapshotTree = { @@ -662,6 +702,26 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext.updateUsedRoutes([""]); assert.strictEqual( dataStoreSummarizerNode?.isReferenced(), true, "Data store should now be referenced"); + } + it("can successfully update referenced state from format version 0", () => { + dataStoreAttributes = { + pkg: "TestDataStore1", + }; + updateReferencedStateTest(); + }); + it("can successfully update referenced state from format version 1", () => { + dataStoreAttributes = { + pkg: "[\"TestDataStore1\"]", + snapshotFormatVersion: "0.1", + }; + updateReferencedStateTest(); + }); + it("can successfully update referenced state from format version 2", () => { + dataStoreAttributes = { + pkg: "[\"TestDataStore1\"]", + summaryFormatVersion: 2, + }; + updateReferencedStateTest(); }); }); }); From 74d00f8b9d1abbfb1c286ad6525f65adf1d99a1e Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 18 Mar 2021 20:29:42 -0400 Subject: [PATCH 30/35] Do not write in new format version unless isolated channels are enabled --- .../container-runtime/src/containerRuntime.ts | 15 +++---- .../container-runtime/src/dataStoreContext.ts | 7 ++- .../container-runtime/src/summaryFormat.ts | 43 +++++++++++-------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/packages/runtime/container-runtime/src/containerRuntime.ts b/packages/runtime/container-runtime/src/containerRuntime.ts index 84a3296a0b6e..7da13d07148c 100644 --- a/packages/runtime/container-runtime/src/containerRuntime.ts +++ b/packages/runtime/container-runtime/src/containerRuntime.ts @@ -1006,18 +1006,15 @@ export class ContainerRuntime extends TypedEventEmitter */ public async snapshot(): Promise { const root: ITree = { entries: [] }; + const entries = await this.dataStores.snapshot(); if (this.disableIsolatedChannels) { - root.entries = root.entries.concat(await this.dataStores.snapshot()); + root.entries = root.entries.concat(entries); } else { - root.entries.push(new TreeTreeEntry( - channelsTreeName, - { entries: await this.dataStores.snapshot() }, - )); + root.entries.push(new TreeTreeEntry(channelsTreeName, { entries })); + root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata()))); } - root.entries.push(new BlobTreeEntry(metadataBlobName, JSON.stringify(this.formMetadata()))); - if (this.chunkMap.size > 0) { root.entries.push(new BlobTreeEntry(chunksBlobName, JSON.stringify([...this.chunkMap]))); } @@ -1026,7 +1023,9 @@ export class ContainerRuntime extends TypedEventEmitter } private addContainerBlobsToSummary(summaryTree: ISummaryTreeWithStats) { - addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata())); + if (!this.disableIsolatedChannels) { + addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(this.formMetadata())); + } if (this.chunkMap.size > 0) { const content = JSON.stringify([...this.chunkMap]); addBlobToSummary(summaryTree, chunksBlobName, content); diff --git a/packages/runtime/container-runtime/src/dataStoreContext.ts b/packages/runtime/container-runtime/src/dataStoreContext.ts index 5e336c43f55d..4291c5c79f43 100644 --- a/packages/runtime/container-runtime/src/dataStoreContext.ts +++ b/packages/runtime/container-runtime/src/dataStoreContext.ts @@ -76,11 +76,14 @@ function createAttributes( disableIsolatedChannels: boolean, ): WriteFluidDataStoreAttributes { const stringifiedPkg = JSON.stringify(pkg); - return { + return disableIsolatedChannels ? { + pkg: stringifiedPkg, + snapshotFormatVersion: "0.1", + isRootDataStore, + } : { pkg: stringifiedPkg, summaryFormatVersion: 2, isRootDataStore, - disableIsolatedChannels: disableIsolatedChannels || undefined, }; } export function createAttributesBlob( diff --git a/packages/runtime/container-runtime/src/summaryFormat.ts b/packages/runtime/container-runtime/src/summaryFormat.ts index 63aa61cd7b81..9a4c8ad41ac3 100644 --- a/packages/runtime/container-runtime/src/summaryFormat.ts +++ b/packages/runtime/container-runtime/src/summaryFormat.ts @@ -6,9 +6,11 @@ import { SummaryType } from "@fluidframework/protocol-definitions"; import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions"; -interface IFluidDataStoreAttributes1 { +type OmitAttributesVersions = Omit; +interface IFluidDataStoreAttributes0 { + readonly snapshotFormatVersion?: undefined; + readonly summaryFormatVersion?: undefined; pkg: string; - readonly snapshotFormatVersion?: "0.1"; /** * This tells whether a data store is root. Root data stores are never collected. * Non-root data stores may be collected if they are not used. If this is not present, default it to @@ -16,10 +18,20 @@ interface IFluidDataStoreAttributes1 { */ readonly isRootDataStore?: boolean; } -interface IFluidDataStoreAttributes2 extends IFluidDataStoreAttributes1 { +interface IFluidDataStoreAttributes1 extends OmitAttributesVersions { + readonly snapshotFormatVersion: "0.1"; + readonly summaryFormatVersion?: undefined; +} +interface IFluidDataStoreAttributes2 extends OmitAttributesVersions { + /** Switch from snapshotFormatVersion to summaryFormatVersion */ readonly snapshotFormatVersion?: undefined; readonly summaryFormatVersion: 2; - /** True if channels are not isolated in .channels subtrees, otherwise isolated. */ + /** + * True if channels are not isolated in .channels subtrees, otherwise isolated. + * This is required in both datastore attributes as well as the root container, + * because reused summary handles may cause different format versions in each + * datastore subtree within the summary. + */ readonly disableIsolatedChannels?: true; } /** @@ -27,11 +39,14 @@ interface IFluidDataStoreAttributes2 extends IFluidDataStoreAttributes1 { * store like the package, snapshotFormatVersion to take different decisions based on a particular * snapshotFormatVersion. */ -export type ReadFluidDataStoreAttributes = IFluidDataStoreAttributes1 | IFluidDataStoreAttributes2; -export type WriteFluidDataStoreAttributes = IFluidDataStoreAttributes2; +export type ReadFluidDataStoreAttributes = + | IFluidDataStoreAttributes0 + | IFluidDataStoreAttributes1 + | IFluidDataStoreAttributes2; +export type WriteFluidDataStoreAttributes = IFluidDataStoreAttributes1 | IFluidDataStoreAttributes2; export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttributes): number { - if ("summaryFormatVersion" in attributes && attributes.summaryFormatVersion !== undefined) { + if (attributes.summaryFormatVersion) { /** * Version 2+: Introduces .channels trees for isolation of * channel trees from data store objects. @@ -51,12 +66,9 @@ export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttribu return 0; } +// eslint-disable-next-line prefer-arrow/prefer-arrow-functions export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): boolean { - const version = getAttributesFormatVersion(attributes); - if (version < 2) { - return false; - } - return !("disableIsolatedChannels" in attributes && attributes.disableIsolatedChannels); + return !!attributes.summaryFormatVersion && !attributes.disableIsolatedChannels; } export interface IContainerRuntimeMetadata { @@ -81,12 +93,9 @@ export const metadataBlobName = ".metadata"; export const chunksBlobName = ".chunks"; export const blobsTreeName = ".blobs"; +// eslint-disable-next-line prefer-arrow/prefer-arrow-functions export function rootHasIsolatedChannels(metadata: IContainerRuntimeMetadata | undefined): boolean { - const version = getMetadataFormatVersion(metadata); - if (version < 1) { - return false; - } - return !(metadata && "disableIsolatedChannels" in metadata && metadata.disableIsolatedChannels); + return !!metadata && !metadata.disableIsolatedChannels; } export const protocolTreeName = ".protocol"; From 16d9cf2f5bccf9404e6ece8d44b539bb4c54f0db Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 18 Mar 2021 20:46:47 -0400 Subject: [PATCH 31/35] Revert snapshot test generation --- packages/test/snapshots/content | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/snapshots/content b/packages/test/snapshots/content index 8f3ed57990af..8229cfcffd67 160000 --- a/packages/test/snapshots/content +++ b/packages/test/snapshots/content @@ -1 +1 @@ -Subproject commit 8f3ed57990af852df66fd3d9b40efed1f906c72d +Subproject commit 8229cfcffd675b59da8172b6f7e7ae8df813a9b6 From bda90e83dd6925cd508992883c4602e2c2028307 Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 18 Mar 2021 21:05:44 -0400 Subject: [PATCH 32/35] Fix summaries test --- .../src/test/summaries.spec.ts | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts index 53d22bcd606b..4f41bb7e4010 100644 --- a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts @@ -4,13 +4,13 @@ */ import { ContainerRuntimeFactoryWithDefaultDataStore, DataObject, DataObjectFactory } from "@fluidframework/aqueduct"; -import { assert, TelemetryNullLogger } from "@fluidframework/common-utils"; +import { assert, bufferToString, TelemetryNullLogger } from "@fluidframework/common-utils"; import { IContainer } from "@fluidframework/container-definitions"; import { ContainerRuntime, IContainerRuntimeOptions } from "@fluidframework/container-runtime"; import { IFluidCodeDetails } from "@fluidframework/core-interfaces"; import { SharedDirectory, SharedMap } from "@fluidframework/map"; import { SharedMatrix } from "@fluidframework/matrix"; -import { SummaryType } from "@fluidframework/protocol-definitions"; +import { ISummaryBlob, SummaryType } from "@fluidframework/protocol-definitions"; import { channelsTreeName } from "@fluidframework/runtime-definitions"; import { requestFluidObject } from "@fluidframework/runtime-utils"; import { SharedObjectSequence } from "@fluidframework/sequence"; @@ -80,6 +80,11 @@ async function createContainer(runtimeOptions: Omit { it("Should generate summary tree", async () => { const { container, opProcessingController } = await createContainer({ disableIsolatedChannels: false }); @@ -106,6 +111,9 @@ describe("Summaries", () => { assert(!summary.unreferenced, "Root summary should be referenced."); assert(summary.tree[".metadata"]?.type === SummaryType.Blob, "Expected .metadata blob in summary root."); + const metadata = readBlobContent(summary.tree[".metadata"].content) as Record; + assert(metadata.summaryFormatVersion === 1, "Metadata blob should have summaryFormatVersion 1"); + assert(metadata.disableIsolatedChannels === undefined, "Unexpected metadata blob disableIsolatedChannels"); const channelsTree = summary.tree[channelsTreeName]; assert(channelsTree?.type === SummaryType.Tree, "Expected .channels tree in summary root."); @@ -116,6 +124,11 @@ describe("Summaries", () => { assert(defaultDataStoreNode.tree[".component"]?.type === SummaryType.Blob, "Expected .component blob in default data store summary tree."); const dataStoreChannelsTree = defaultDataStoreNode.tree[channelsTreeName]; + const attributes = readBlobContent(defaultDataStoreNode.tree[".component"].content) as Record; + assert(attributes.snapshotFormatVersion === undefined, "Unexpected datastore attributes snapshotFormatVersion"); + assert(attributes.summaryFormatVersion === 2, "Datastore attributes summaryFormatVersion should be 2"); + assert(attributes.disableIsolatedChannels === undefined, + "Unexpected datastore attributes disableIsolatedChannels"); assert(dataStoreChannelsTree?.type === SummaryType.Tree, "Expected .channels tree in default data store."); const defaultDdsNode = dataStoreChannelsTree.tree.root; @@ -155,7 +168,7 @@ describe("Summaries", () => { // Validate summary assert(!summary.unreferenced, "Root summary should be referenced."); - assert(summary.tree[".metadata"]?.type === SummaryType.Blob, "Expected .metadata blob in summary root."); + assert(summary.tree[".metadata"] === undefined, "Unexpected .metadata blob in summary root."); assert(summary.tree[channelsTreeName] === undefined, "Unexpected .channels tree in summary root."); @@ -164,6 +177,11 @@ describe("Summaries", () => { assert(!defaultDataStoreNode.unreferenced, "Default data store should be referenced."); assert(defaultDataStoreNode.tree[".component"]?.type === SummaryType.Blob, "Expected .component blob in default data store summary tree."); + const attributes = readBlobContent(defaultDataStoreNode.tree[".component"].content) as Record; + assert(attributes.snapshotFormatVersion === "0.1", "Datastore attributes snapshotFormatVersion should be 0.1"); + assert(attributes.summaryFormatVersion === undefined, "Unexpected datastore attributes summaryFormatVersion"); + assert(attributes.disableIsolatedChannels === undefined, + "Unexpected datastore attributes disableIsolatedChannels"); assert(defaultDataStoreNode.tree[channelsTreeName] === undefined, "Unexpected .channels tree in default data store."); From a5f06cc597cd2de8964d374959540d8ec2c08f0c Mon Sep 17 00:00:00 2001 From: Arin Date: Thu, 18 Mar 2021 22:13:16 -0400 Subject: [PATCH 33/35] Run all 8 combinations for intiialization --- .../src/test/dataStoreContext.spec.ts | 222 +++++++----------- 1 file changed, 83 insertions(+), 139 deletions(-) diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index ea996667b6df..ba16b7a2dff8 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -294,17 +294,9 @@ describe("Data Store Context Tests", () => { let dataStoreAttributes: ReadFluidDataStoreAttributes; const storage: Partial = {}; let scope: IFluidObject; - let containerRuntime: ContainerRuntime; let summarizerNode: IRootSummarizerNodeWithGC; - beforeEach(async () => { - summarizerNode = createRootSummarizerNodeWithGC( - new TelemetryNullLogger(), - (() => undefined) as unknown as SummarizeInternalFn, - 0, - 0); - summarizerNode.startSummary(0, new TelemetryNullLogger()); - + function mockContainerRuntime(disableIsolatedChannels = true): ContainerRuntime { const factory: { [key: string]: any } = {}; factory.IFluidDataStoreFactory = factory; factory.instantiateDataStore = @@ -314,14 +306,24 @@ describe("Data Store Context Tests", () => { registry.get = async (pkg) => Promise.resolve(factory); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - containerRuntime = { + return { + disableIsolatedChannels, IFluidDataStoreRegistry: registry, notifyDataStoreInstantiated: (c) => { }, on: (event, listener) => { }, } as ContainerRuntime; + } + + beforeEach(async () => { + summarizerNode = createRootSummarizerNodeWithGC( + new TelemetryNullLogger(), + (() => undefined) as unknown as SummarizeInternalFn, + 0, + 0); + summarizerNode.startSummary(0, new TelemetryNullLogger()); }); - describe("Initialization", () => { + describe("Initialization - can correctly initialize and generate attributes", () => { beforeEach(() => { createSummarizerNodeFn = ( summarizeInternal: SummarizeInternalFn, @@ -337,138 +339,80 @@ describe("Data Store Context Tests", () => { getInitialGCSummaryDetailsFn, ); }); - - it("can correctly initialize and generate attributes", async () => { - dataStoreAttributes = { - pkg: JSON.stringify(["TestDataStore1"]), + const pkgName = "TestDataStore1"; + + function testGenerateAttributes(disableIsolatedChannels: boolean, expected: WriteFluidDataStoreAttributes) { + async function testGenerateAttributesCore(attributes: ReadFluidDataStoreAttributes) { + const buffer = stringToBuffer(JSON.stringify(attributes), "utf8"); + const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); + const snapshotTree: ISnapshotTree = { + blobs: { [dataStoreAttributesBlobName]: "fluidDataStoreAttributes" }, + commits: {}, + trees: {}, + }; + + remotedDataStoreContext = new RemotedFluidDataStoreContext( + dataStoreId, + snapshotTree, + mockContainerRuntime(disableIsolatedChannels), + new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), + scope, + createSummarizerNodeFn, + ); + + const isRootNode = await remotedDataStoreContext.isRoot(); + assert.strictEqual(isRootNode, true, "The data store should be root."); + + const summarizeResult = await remotedDataStoreContext.summarize(true /* fullTree */); + assert(summarizeResult.summary.type === SummaryType.Tree, + "summarize should always return a tree when fullTree is true"); + const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; + + const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; + + // Validate that generated attributes are as expected. + assert.deepStrictEqual(contents, expected, "Unexpected datastore attributes written"); + } + + it("reading from latest with isolated channels", async () => testGenerateAttributesCore({ + pkg: JSON.stringify([pkgName]), summaryFormatVersion: 2, isRootDataStore: true, - }; - const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); - const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); - const snapshotTree: ISnapshotTree = { - blobs: { [dataStoreAttributesBlobName]: "fluidDataStoreAttributes" }, - commits: {}, - trees: {}, - }; - - remotedDataStoreContext = new RemotedFluidDataStoreContext( - dataStoreId, - snapshotTree, - containerRuntime, - new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), - scope, - createSummarizerNodeFn, - ); - - const isRootNode = await remotedDataStoreContext.isRoot(); - assert.strictEqual(isRootNode, true, "The data store should be root."); - - remotedDataStoreContext.updateUsedRoutes([""]); - - const summarizeResult = await remotedDataStoreContext.summarize(true /* fullTree */); - assert(summarizeResult.summary.type === SummaryType.Tree, - "summarize should always return a tree when fullTree is true"); - const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; + })); - const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; - assert.strictEqual(contents.pkg, dataStoreAttributes.pkg, "Remote DataStore package does not match."); - assert.strictEqual( - contents.summaryFormatVersion, - dataStoreAttributes.summaryFormatVersion, - "Remote DataStore snapshot version does not match."); - assert.strictEqual( - contents.isRootDataStore, - dataStoreAttributes.isRootDataStore, - "Remote DataStore root state does not match"); - }); - - it("can correctly initialize and generate attributes without version and isRootDataStore", async () => { - dataStoreAttributes = { pkg: "TestDataStore1" }; - const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); - const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); - const snapshotTree: ISnapshotTree = { - blobs: { [dataStoreAttributesBlobName]: "fluidDataStoreAttributes" }, - commits: {}, - trees: {}, - }; - - remotedDataStoreContext = new RemotedFluidDataStoreContext( - dataStoreId, - snapshotTree, - containerRuntime, - new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), - scope, - createSummarizerNodeFn, - ); - - const isRootNode = await remotedDataStoreContext.isRoot(); - assert.strictEqual(isRootNode, true, "The data store should be root."); - - const summarizeResult = await remotedDataStoreContext.summarize(true /* fullTree */); - assert(summarizeResult.summary.type === SummaryType.Tree, - "summarize should always return a tree when fullTree is true"); - const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; - - const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; - assert.strictEqual( - contents.pkg, - JSON.stringify([dataStoreAttributes.pkg]), - "Remote DataStore package does not match."); - assert.strictEqual( - contents.summaryFormatVersion, - 2, - "Remote DataStore snapshot version does not match."); - // Remote context without the isRootDataStore flag in the snapshot should default it to true. - assert.strictEqual(contents.isRootDataStore, true, "Remote DataStore root state does not match."); - }); - - it("can correctly initialize and generate attributes with isolated channels disabled", async () => { - dataStoreAttributes = { - pkg: JSON.stringify(["TestDataStore1"]), + it("reading from latest without isolated channels", async () => testGenerateAttributesCore({ + pkg: JSON.stringify([pkgName]), summaryFormatVersion: 2, isRootDataStore: true, - }; - (containerRuntime as any).disableIsolatedChannels = true; + disableIsolatedChannels: true, + })); - const buffer = stringToBuffer(JSON.stringify(dataStoreAttributes), "utf8"); - const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); - const snapshotTree: ISnapshotTree = { - blobs: { [dataStoreAttributesBlobName]: "fluidDataStoreAttributes" }, - commits: {}, - trees: {}, - }; - - remotedDataStoreContext = new RemotedFluidDataStoreContext( - dataStoreId, - snapshotTree, - containerRuntime, - new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), - scope, - createSummarizerNodeFn, - ); + it("reading from previous snapshot format", async () => testGenerateAttributesCore({ + pkg: JSON.stringify([pkgName]), + snapshotFormatVersion: "0.1", + isRootDataStore: true, + })); - const isRootNode = await remotedDataStoreContext.isRoot(); - assert.strictEqual(isRootNode, true, "The data store should be root."); + it("reading from oldest snapshot format", async () => testGenerateAttributesCore({ pkg: pkgName })); + } - const summarizeResult = await remotedDataStoreContext.summarize(true /* fullTree */); - assert(summarizeResult.summary.type === SummaryType.Tree, - "summarize should always return a tree when fullTree is true"); - const blob = summarizeResult.summary.tree[dataStoreAttributesBlobName] as ISummaryBlob; + describe("writing with isolated channels disabled", () => testGenerateAttributes( + true, /* disableIsolatedChannels */ + { + pkg: JSON.stringify([pkgName]), + snapshotFormatVersion: "0.1", + isRootDataStore: true, + }, + )); - const contents = JSON.parse(blob.content as string) as WriteFluidDataStoreAttributes; - assert.strictEqual( - contents.pkg, - dataStoreAttributes.pkg, - "Remote DataStore package does not match."); - assert.strictEqual( - contents.summaryFormatVersion, - 2, - "Remote DataStore snapshot version does not match."); - // Remote context without the isRootDataStore flag in the snapshot should default it to true. - assert.strictEqual(contents.isRootDataStore, true, "Remote DataStore root state does not match."); - assert.strictEqual(contents.disableIsolatedChannels, true, "Disable isolated channels should be true."); - }); + describe("writing with isolated channels enabled", () => testGenerateAttributes( + false, /* disableIsolatedChannels */ + { + pkg: JSON.stringify([pkgName]), + summaryFormatVersion: 2, + isRootDataStore: true, + }, + )); }); describe("Garbage Collection", () => { @@ -505,7 +449,7 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext = new RemotedFluidDataStoreContext( dataStoreId, snapshotTree, - containerRuntime, + mockContainerRuntime(), new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, @@ -553,7 +497,7 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext = new RemotedFluidDataStoreContext( dataStoreId, snapshotTree, - containerRuntime, + mockContainerRuntime(), new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, @@ -608,7 +552,7 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext = new RemotedFluidDataStoreContext( dataStoreId, snapshotTree, - containerRuntime, + mockContainerRuntime(), new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, @@ -645,7 +589,7 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext = new RemotedFluidDataStoreContext( dataStoreId, snapshotTree, - containerRuntime, + mockContainerRuntime(), new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, @@ -682,7 +626,7 @@ describe("Data Store Context Tests", () => { remotedDataStoreContext = new RemotedFluidDataStoreContext( dataStoreId, snapshotTree, - containerRuntime, + mockContainerRuntime(), new BlobCacheStorageService(storage as IDocumentStorageService, blobCache), scope, createSummarizerNodeFn, From 1032cbea52aca80223d1c12a24d77a95d6504f82 Mon Sep 17 00:00:00 2001 From: Arin Date: Mon, 22 Mar 2021 18:20:15 -0400 Subject: [PATCH 34/35] PR comments --- .../src/test/dataStoreContext.spec.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts index ba16b7a2dff8..4eae167b3d20 100644 --- a/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts +++ b/packages/runtime/container-runtime/src/test/dataStoreContext.spec.ts @@ -341,7 +341,21 @@ describe("Data Store Context Tests", () => { }); const pkgName = "TestDataStore1"; + /** + * Runs the initialization and generate datastore attributes tests with the given write-mode preferences + * and expectations. + * This runs the same test with various summary write and read preferences. Specifically each call of this + * function will run the test 4 times, one for each possible summary format we could be reading from. + * @param disableIsolatedChannels - whether or not to write summary with isolated channels disabled or not + * @param expected - the expected datastore attributes to be generated given the write preference + */ function testGenerateAttributes(disableIsolatedChannels: boolean, expected: WriteFluidDataStoreAttributes) { + /** + * This function is called for each possible base snapshot format version. We want to cover all + * summary format read/write combinations. We only write in latest or -1 version, but we can + * need to be able to read old summary format versions forever. + * @param attributes - datastore attributes that are in the base snapshot we load from + */ async function testGenerateAttributesCore(attributes: ReadFluidDataStoreAttributes) { const buffer = stringToBuffer(JSON.stringify(attributes), "utf8"); const blobCache = new Map([["fluidDataStoreAttributes", buffer]]); @@ -374,26 +388,26 @@ describe("Data Store Context Tests", () => { assert.deepStrictEqual(contents, expected, "Unexpected datastore attributes written"); } - it("reading from latest with isolated channels", async () => testGenerateAttributesCore({ + it("can read from latest with isolated channels", async () => testGenerateAttributesCore({ pkg: JSON.stringify([pkgName]), summaryFormatVersion: 2, isRootDataStore: true, })); - it("reading from latest without isolated channels", async () => testGenerateAttributesCore({ + it("can read from latest without isolated channels", async () => testGenerateAttributesCore({ pkg: JSON.stringify([pkgName]), summaryFormatVersion: 2, isRootDataStore: true, disableIsolatedChannels: true, })); - it("reading from previous snapshot format", async () => testGenerateAttributesCore({ + it("can read from previous snapshot format", async () => testGenerateAttributesCore({ pkg: JSON.stringify([pkgName]), snapshotFormatVersion: "0.1", isRootDataStore: true, })); - it("reading from oldest snapshot format", async () => testGenerateAttributesCore({ pkg: pkgName })); + it("can read from oldest snapshot format", async () => testGenerateAttributesCore({ pkg: pkgName })); } describe("writing with isolated channels disabled", () => testGenerateAttributes( @@ -647,12 +661,14 @@ describe("Data Store Context Tests", () => { assert.strictEqual( dataStoreSummarizerNode?.isReferenced(), true, "Data store should now be referenced"); } + it("can successfully update referenced state from format version 0", () => { dataStoreAttributes = { pkg: "TestDataStore1", }; updateReferencedStateTest(); }); + it("can successfully update referenced state from format version 1", () => { dataStoreAttributes = { pkg: "[\"TestDataStore1\"]", @@ -660,6 +676,7 @@ describe("Data Store Context Tests", () => { }; updateReferencedStateTest(); }); + it("can successfully update referenced state from format version 2", () => { dataStoreAttributes = { pkg: "[\"TestDataStore1\"]", From be226477165106228a1e019728d5b9b4eb72e08c Mon Sep 17 00:00:00 2001 From: Arin Date: Mon, 22 Mar 2021 19:28:02 -0400 Subject: [PATCH 35/35] Fix build --- .../src/test/summaries.spec.ts | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts index 0d92f59dbca1..2bfd99103011 100644 --- a/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/summaries.spec.ts @@ -36,10 +36,7 @@ class TestDataObject extends DataObject { async function createContainer( provider: ITestObjectProvider, runtimeOptions: Omit, -): Promise<{ - container: IContainer; - opProcessingController: OpProcessingController; -}> { +): Promise { const documentId = createDocumentId(); const codeDetails: IFluidCodeDetails = { package: "summarizerTestPackage", @@ -97,14 +94,11 @@ describeNoCompat("Summaries", (getTestObjectProvider) => { }); it("Should generate summary tree", async () => { - const { container, opProcessingController } = await createContainer( - provider, - { disableIsolatedChannels: false }, - ); + const container = await createContainer(provider, { disableIsolatedChannels: false }); const defaultDataStore = await requestFluidObject(container, defaultDataStoreId); const containerRuntime = defaultDataStore.getContext().containerRuntime as ContainerRuntime; - await opProcessingController.process(); + await provider.ensureSynchronized(); const { gcData, stats, summary } = await containerRuntime.summarize({ runGC: false, @@ -158,10 +152,7 @@ describeNoCompat("Summaries", (getTestObjectProvider) => { }); it("Should generate summary tree with isolated channels disabled", async () => { - const { container, opProcessingController } = await createContainer( - provider, - { disableIsolatedChannels: true }, - ); + const container = await createContainer(provider, { disableIsolatedChannels: true }); const defaultDataStore = await requestFluidObject(container, defaultDataStoreId); const containerRuntime = defaultDataStore.getContext().containerRuntime as ContainerRuntime;