From b1cbab64929c7fdf4a5dd23f07fcf48f92fb4ab4 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Fri, 3 Dec 2021 14:20:53 -0800 Subject: [PATCH 01/18] General Tree Works --- WordpressSync/endpoints.json | 2 +- package-lock.json | 190 +++++------ wordpress-to-github/common/index.js | 75 ----- wordpress-to-github/gitTreeCommon/index.js | 352 --------------------- wordpress-to-github/index.js | 106 +++---- wordpress-to-github/package.json | 3 +- 6 files changed, 128 insertions(+), 600 deletions(-) delete mode 100644 wordpress-to-github/gitTreeCommon/index.js diff --git a/WordpressSync/endpoints.json b/WordpressSync/endpoints.json index 85c1bc0..239ee1c 100644 --- a/WordpressSync/endpoints.json +++ b/WordpressSync/endpoints.json @@ -10,7 +10,7 @@ "name": "Covid 19 website", "description": "Using data to create test output.", "enabled": false, - "enabledLocal": false, + "enabledLocal": true, "ReportingChannel_Slack": "C01H6RB99E2", "WordPressSource": { "url": "https://as-go-covid19-d-001.azurewebsites.net", diff --git a/package-lock.json b/package-lock.json index 5ec0faa..fbba292 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,21 @@ "prettier": "^2.5.0" } }, + "node_modules/@cagov/github-tree-push": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.0.1.tgz", + "integrity": "sha512-JPrb48dO8n2gkIKtdn0jFxY7oSUS72y5Udvy/0ILMR28NIxTwhcYGsv9L33hBc3YLIPuB0Tw/GoX6dzObBYttw==", + "dependencies": { + "fetch-retry": "^5.0.1", + "node-fetch": "^2.6.6", + "sha1": "^1.1.1" + } + }, + "node_modules/@cagov/github-tree-push/node_modules/fetch-retry": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.1.tgz", + "integrity": "sha512-ENMet8exSRjJO8I5PoKChm0psngw0VrBOD9fUH4YO7sE7Dpqnb0/iFEGHZn633z6MPNVJv4zZNrGgQ4lmQt1nw==" + }, "node_modules/@cagov/slack-connector": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@cagov/slack-connector/-/slack-connector-0.0.1.tgz", @@ -207,13 +222,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "node_modules/axios": { - "version": "0.21.4", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", - "dependencies": { - "follow-redirects": "^1.14.0" - } - }, "node_modules/balanced-match": { "version": "1.0.2", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", @@ -309,13 +317,6 @@ "node": "*" } }, - "node_modules/debug": { - "version": "2.6.9", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" - } - }, "node_modules/deep-is": { "version": "0.1.4", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", @@ -664,24 +665,6 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, - "node_modules/follow-redirects": { - "version": "1.14.1", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", @@ -692,16 +675,6 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, - "node_modules/github-api": { - "version": "3.4.0", - "integrity": "sha512-2yYqYS6Uy4br1nw0D3VrlYWxtGTkUhIZrumBrcBwKdBOzMT8roAe8IvI6kjIOkxqxapKR5GkEsHtz3Du/voOpA==", - "dependencies": { - "axios": "^0.21.1", - "debug": "^2.2.0", - "js-base64": "^2.1.9", - "utf8": "^2.1.1" - } - }, "node_modules/glob": { "version": "7.2.0", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", @@ -829,10 +802,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "node_modules/js-base64": { - "version": "2.6.4", - "integrity": "sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==" - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -909,18 +878,18 @@ "node": "*" } }, - "node_modules/ms": { - "version": "2.0.0", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, "node_modules/natural-compare": { "version": "1.4.0", "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, "node_modules/node-fetch": { - "version": "2.6.1", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { "node": "4.x || >=6.0.0" } @@ -1171,6 +1140,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "node_modules/type-check": { "version": "0.4.0", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", @@ -1203,15 +1177,25 @@ "punycode": "^2.1.0" } }, - "node_modules/utf8": { - "version": "2.1.2", - "integrity": "sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY=" - }, "node_modules/v8-compile-cache": { "version": "2.3.0", "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", @@ -1249,14 +1233,30 @@ "version": "0.2.0", "license": "MIT", "dependencies": { + "@cagov/github-tree-push": "^0.0.1", "fetch-retry": "^4.1.1", - "github-api": "^3.4.0", - "node-fetch": "^2.6.1", - "sha1": "^1.1.1" + "node-fetch": "^2.6.1" } } }, "dependencies": { + "@cagov/github-tree-push": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.0.1.tgz", + "integrity": "sha512-JPrb48dO8n2gkIKtdn0jFxY7oSUS72y5Udvy/0ILMR28NIxTwhcYGsv9L33hBc3YLIPuB0Tw/GoX6dzObBYttw==", + "requires": { + "fetch-retry": "^5.0.1", + "node-fetch": "^2.6.6", + "sha1": "^1.1.1" + }, + "dependencies": { + "fetch-retry": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.1.tgz", + "integrity": "sha512-ENMet8exSRjJO8I5PoKChm0psngw0VrBOD9fUH4YO7sE7Dpqnb0/iFEGHZn633z6MPNVJv4zZNrGgQ4lmQt1nw==" + } + } + }, "@cagov/slack-connector": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@cagov/slack-connector/-/slack-connector-0.0.1.tgz", @@ -1269,10 +1269,9 @@ "@cagov/wordpress-to-github": { "version": "file:wordpress-to-github", "requires": { + "@cagov/github-tree-push": "^0.0.1", "fetch-retry": "^4.1.1", - "github-api": "^3.4.0", - "node-fetch": "^2.6.1", - "sha1": "^1.1.1" + "node-fetch": "^2.6.1" } }, "@es-joy/jsdoccomment": { @@ -1404,13 +1403,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "axios": { - "version": "0.21.4", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", - "requires": { - "follow-redirects": "^1.14.0" - } - }, "balanced-match": { "version": "1.0.2", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", @@ -1482,13 +1474,6 @@ "version": "0.0.2", "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" }, - "debug": { - "version": "2.6.9", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, "deep-is": { "version": "0.1.4", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", @@ -1744,10 +1729,6 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, - "follow-redirects": { - "version": "1.14.1", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" - }, "fs.realpath": { "version": "1.0.0", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", @@ -1758,16 +1739,6 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, - "github-api": { - "version": "3.4.0", - "integrity": "sha512-2yYqYS6Uy4br1nw0D3VrlYWxtGTkUhIZrumBrcBwKdBOzMT8roAe8IvI6kjIOkxqxapKR5GkEsHtz3Du/voOpA==", - "requires": { - "axios": "^0.21.1", - "debug": "^2.2.0", - "js-base64": "^2.1.9", - "utf8": "^2.1.1" - } - }, "glob": { "version": "7.2.0", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", @@ -1859,10 +1830,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "js-base64": { - "version": "2.6.4", - "integrity": "sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==" - }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -1924,18 +1891,18 @@ "brace-expansion": "^1.1.7" } }, - "ms": { - "version": "2.0.0", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, "natural-compare": { "version": "1.4.0", "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, "node-fetch": { - "version": "2.6.1", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "requires": { + "whatwg-url": "^5.0.0" + } }, "once": { "version": "1.4.0", @@ -2108,6 +2075,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "type-check": { "version": "0.4.0", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", @@ -2131,15 +2103,25 @@ "punycode": "^2.1.0" } }, - "utf8": { - "version": "2.1.2", - "integrity": "sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY=" - }, "v8-compile-cache": { "version": "2.3.0", "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", diff --git a/wordpress-to-github/common/index.js b/wordpress-to-github/common/index.js index 1fbaa91..4948e4f 100644 --- a/wordpress-to-github/common/index.js +++ b/wordpress-to-github/common/index.js @@ -1,10 +1,6 @@ // @ts-check const crypto = require("crypto"); const apiPath = "/wp-json/wp/v2/"; -const { - gitHubBlobPredictShaFromBuffer, - GithubTreeRow -} = require("../gitTreeCommon"); const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { retries: 3, retryDelay: 5000, @@ -392,76 +388,6 @@ const wrapInFileMeta = ( data }); -/** - * A custom Github function to check for file exists - * - * @param {{ _request: function( string, *, *) : Promise<*> }} myRepo - * @param {string} path - * @param {undefined} [data] - * @param {function(any, boolean, any) : void} [cb] - */ -function githubDoesFileExist(myRepo, path, data, cb) { - return myRepo._request("HEAD", path, data).then( - (/** @type {any} */ response) => { - if (cb) { - cb(null, true, response); - } - return true; - }, - (/** @type {{ response: { status: number } }} */ response) => { - if (response.response.status === 404) { - if (cb) { - cb(null, false, response); - } - return false; - } - - if (cb) { - // @ts-ignore - cb(response); - } - throw response; - } - ); -} - -/** - * Syncs a binary file with Github, by adding the blob if its not already there and then updating the sha in the tree - * - * @param {string} wordpress_url - * @param {*} gitRepo - * @param {GithubTreeRow[]} mediaTree - * @param {EndpointConfigData} endpoint - */ -const syncBinaryFile = async (wordpress_url, gitRepo, mediaTree, endpoint) => { - console.log(`Downloading...${wordpress_url}`); - const fetchResponse = await fetchRetry(wordpress_url, { method: "Get" }); - const blob = await fetchResponse.arrayBuffer(); - const buffer = Buffer.from(blob); - - let sha = gitHubBlobPredictShaFromBuffer(buffer); - - const exists = await githubDoesFileExist( - gitRepo, - `/repos/${gitRepo.__fullname}/git/blobs/${sha}` - ); - if (!exists) { - const blobResult = await gitRepo.createBlob(buffer); - sha = blobResult.data.sha; //should be the same, but just in case - } - - //swap in the new blob sha here. If the sha matches something already there it will be determined on server. - const treeNode = mediaTree.find( - x => - x.path === - `${endpoint.MediaPath}/${pathFromMediaSourceUrl(wordpress_url)}` - ); - if (treeNode) { - delete treeNode.content; - treeNode.sha = sha; - } -}; - /** * deletes properties in the list * @@ -528,7 +454,6 @@ const addMediaSection = (endpoint, mediaMap, jsonData, HTML) => { module.exports = { ensureStringStartsWith, removeExcludedProperties, - syncBinaryFile, wrapInFileMeta, commonMeta, WpApi_GetCacheItem_ByObjectType, diff --git a/wordpress-to-github/gitTreeCommon/index.js b/wordpress-to-github/gitTreeCommon/index.js deleted file mode 100644 index 8530424..0000000 --- a/wordpress-to-github/gitTreeCommon/index.js +++ /dev/null @@ -1,352 +0,0 @@ -// @ts-check -// Updated 2021-10-18 - -const nowPacTime = (/** @type {Intl.DateTimeFormatOptions} */ options) => - new Date().toLocaleString("en-CA", { - timeZone: "America/Los_Angeles", - ...options - }); -const todayDateString = () => - nowPacTime({ year: "numeric", month: "2-digit", day: "2-digit" }); -const todayTimeString = () => - nowPacTime({ - hour12: false, - hour: "2-digit", - minute: "2-digit", - second: "2-digit" - }).replace(/:/g, "-"); -/** - * Halts processing for a set time - * - * @param {number} ms milliseconds to sleep (1000 = 1s) - */ -const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); - -//Git generates the SHA by concatenating a header in the form of blob {content.length} {null byte} and the contents of your file -const sha1 = require("sha1"); -/** - * Returns a Github equivalent sha hash for any given content - * see https://git-scm.com/book/en/v2/Git-Internals-Git-Objects - * - * @param {string} content string content to hash - * @returns SHA Hash that would be used on Github for the given content - */ -const gitHubBlobPredictSha = content => - sha1(`blob ${Buffer.byteLength(content)}\0${content}`); - -/** - * Returns a Github equivalent sha hash for any given content - * see https://git-scm.com/book/en/v2/Git-Internals-Git-Objects - * - * @param {Buffer} buffer buffer to hash - * @returns SHA Hash that would be used on Github for the given content - */ -const gitHubBlobPredictShaFromBuffer = buffer => - sha1( - Buffer.concat([Buffer.from(`blob ${buffer.byteLength}\0`, "utf8"), buffer]) - ); -/** - * @typedef {object} GithubTreeRow - * @property {string} path - * @property {string} mode usually '100644' - * @property {string} type usually 'blob' - * @property {string} [sha] - * @property {string} [content] - */ - -/** - * @typedef {object} GithubPullRequest - * @property {string} html_url - * @property {number} number - * @property {{ref:string}} head - */ - -/** - * @typedef {object} GithubCommit - * @property {string} sha - * @property {string} html_url - * @property {string} message - */ - -/** - * @typedef {object} GithubCompareFile - * @property {string} filename - * @property {string} status - */ - -/** - * @typedef {object} CommitReport - * @property {GithubCommit} Commit - * @property {GithubCompareFile[]} Files - */ - -/** - * Creates a gitHub Tree array, skipping duplicates based on the outputpath - * - * @param {*} gitRepo from github-api - * @param {string} masterBranch usually "master" or "main" - * @param {Map} filesMap contains the data to push - * @param {string} outputPath the root path for all files - * @param {boolean} [cleanoutputPath] true to delete all unmatched files in outputPath - */ -const createTreeFromFileMap = async ( - gitRepo, - masterBranch, - filesMap, - outputPath, - cleanoutputPath -) => { - let treeUrl = ""; - if (outputPath) { - //Path Tree - - const pathRootTree = outputPath.split("/").slice(0, -1).join("/"); //gets the parent folder to the output path - /** @type {GithubTreeRow[]} */ - const rootTree = (await gitRepo.getSha(masterBranch, pathRootTree)).data; - const referenceTreeRow = rootTree.find(f => f.path === outputPath); - - if (referenceTreeRow) { - treeUrl = `${referenceTreeRow.sha}?recursive=true`; - } - } else { - //Root Tree - treeUrl = masterBranch; - } - - const referenceTree = /** @type {{data:{tree:GithubTreeRow[]}}}} */ ( - await gitRepo.getTree(treeUrl) - ).data.tree.filter(x => x.type === "blob"); - - /** @type {GithubTreeRow[]} */ - const targetTree = []; - //Tree parts... - //https://docs.github.com/en/free-pro-team@latest/rest/reference/git#create-a-tree - const mode = "100644"; //code for tree blob - const type = "blob"; - - for (const [key, value] of filesMap) { - let existingFile = referenceTree.find(x => x.path === key); - if (existingFile) { - existingFile["found"] = true; - } - if (value) { - //ignoring files with null value - let content = - typeof value === "string" ? value : JSON.stringify(value, null, 2); - - if (!existingFile || existingFile.sha !== gitHubBlobPredictSha(content)) { - let path = outputPath ? `${outputPath}/${key}` : key; - - targetTree.push({ - path, - content, - mode, - type - }); - } - } - } - - if (cleanoutputPath) { - //process deletes - for (const delme of referenceTree.filter(x => !x["found"])) { - let path = outputPath ? `${outputPath}/${delme.path}` : delme.path; - - targetTree.push({ - path, - mode, - type, - sha: null //will trigger a delete - }); - } - } - - return targetTree; -}; - -/** - * return a new PR if the tree has changes - * - * @param {*} gitRepo from github-api - * @param {string} masterBranch usually "master" or "main" - * @param {GithubTreeRow[]} tree from createTreeFromFileMap - * @param {string} PrTitle the name of the new branch to create - * @param {{name:string,email:string}} committer Github Name/Email - * @param {boolean} commit_only true if skipping the PR process and just making a commit - * @returns the new PR - */ -const CommitOrPrIfChanged = async ( - gitRepo, - masterBranch, - tree, - PrTitle, - committer, - commit_only -) => { - if (!tree.length) { - return null; - } - - const newBranchName = `${PrTitle}-${todayTimeString()}`.replace(/ /g, "_"); - let treeParts = [tree]; - const totalRows = tree.length; - - console.log(`Tree data is ${Buffer.byteLength(JSON.stringify(tree))} bytes`); - - //Split the tree into allowable sizes - let evalIndex = 0; - while (evalIndex < treeParts.length) { - if (JSON.stringify(treeParts[evalIndex]).length > 9000000) { - let half = Math.ceil(treeParts[evalIndex].length / 2); - treeParts.unshift(treeParts[evalIndex].splice(0, half)); - } else { - evalIndex++; - } - } - - //Grab the starting point for a fresh tree - /** @type {{data:{object:{sha:string}}}} */ - const refResult = await gitRepo.getRef(`heads/${masterBranch}`); - const baseSha = refResult.data.object.sha; - - //Loop through adding items to the tree - let createTreeResult = { data: { sha: baseSha } }; - let rowCount = 0; - for (let treePart of treeParts) { - rowCount += treePart.length; - console.log( - `Creating tree for ${PrTitle} - ${rowCount}/${totalRows} items` - ); - - createTreeResult = await gitRepo.createTree( - treePart, - createTreeResult.data.sha - ); - } - - //Create a commit the maps to all the tree changes - /** @type {GithubCommit} */ - const commitResult = ( - await gitRepo.commit(baseSha, createTreeResult.data.sha, PrTitle, committer) - ).data; - const commitSha = commitResult.sha; - - //Compare the proposed commit with the trunk (master) branch - /** @type {{files:GithubCompareFile[]}} */ - const compare = (await gitRepo.compareBranches(baseSha, commitSha)).data; - if (compare.files.length) { - console.log(`${compare.files.length} changes.`); - - if (commit_only) { - console.log(`Commit created - ${commitResult.html_url}`); - await gitRepo.updateHead(`heads/${masterBranch}`, commitSha); - - return { - PullRequest: null, - Commit: commitResult, - Files: compare.files - }; - } else { - //Create a new branch and assign this commit to it, return the new branch. - await gitRepo.createBranch(masterBranch, newBranchName); - await gitRepo.updateHead(`heads/${newBranchName}`, commitSha); - - /** @type {GithubPullRequest} */ - const Pr = ( - await gitRepo.createPullRequest({ - title: PrTitle, - head: newBranchName, - base: masterBranch - }) - ).data; - - console.log(`PR created - ${Pr.html_url}`); - - return { - PullRequest: Pr, - Commit: commitResult, - Files: compare.files - }; - } - } else { - console.log("no changes"); - return null; - } -}; - -/** - * return a new PR if the tree has changes - * - * @param {*} gitRepo from github-api - * @param {string} masterBranch usually "master" or "main" - * @param {GithubTreeRow[]} tree from createTreeFromFileMap - * @param {string} PrTitle the name of the new branch to create - * @param {{name:string,email:string}} committer Github Name/Email - * @param {boolean} [commit_only] deprecated - * @returns the new PR - */ -const PrIfChanged = async ( - gitRepo, - masterBranch, - tree, - PrTitle, - committer, - commit_only -) => { - if (commit_only !== undefined) { - throw new Error("commit_only is deprecated"); - } - - const PrResult = await CommitOrPrIfChanged( - gitRepo, - masterBranch, - tree, - PrTitle, - committer, - false - ); - - if (PrResult) return PrResult.PullRequest; -}; - -/** - * return a new PR if the tree has changes - * - * @param {*} gitRepo from github-api - * @param {string} masterBranch usually "master" or "main" - * @param {GithubTreeRow[]} tree from createTreeFromFileMap - * @param {string} PrTitle the name of the new branch to create - * @param {{name:string,email:string}} committer Github Name/Email - * @returns the new PR - */ -const CommitIfChanged = async ( - gitRepo, - masterBranch, - tree, - PrTitle, - committer -) => { - /** @type {CommitReport} */ - const Result = await CommitOrPrIfChanged( - gitRepo, - masterBranch, - tree, - PrTitle, - committer, - true - ); - - return Result; -}; - -module.exports = { - createTreeFromFileMap, - PrIfChanged, - CommitIfChanged, - todayDateString, - todayTimeString, - nowPacTime, - sleep, - gitHubBlobPredictSha, - gitHubBlobPredictShaFromBuffer -}; diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index f314365..1b402c0 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -1,14 +1,8 @@ // @ts-check -const GitHub = require("github-api"); -const { - createTreeFromFileMap, - CommitIfChanged, - CommitReport -} = require("./gitTreeCommon"); +const { GitHubTreePush, GithubCompare } = require("@cagov/github-tree-push"); const { ensureStringStartsWith, removeExcludedProperties, - syncBinaryFile, wrapInFileMeta, commonMeta, WpApi_GetCacheItem_ByObjectType, @@ -47,33 +41,6 @@ const fieldMetaReference = { const updateCache = new Map(); const cacheObjects = ["media", "posts", "pages"]; -/** - * process a Wordpress endpoint and place the data in GitHub - * - * @param {GitHubTarget} gitHubTarget - * @param {GitHubCredentials} gitHubCredentials - */ -const getRemoteConfig = async (gitHubTarget, gitHubCredentials) => { - const gitModule = new GitHub(gitHubCredentials); - - // @ts-ignore - const gitRepo = await gitModule.getRepo( - gitHubTarget.Owner, - gitHubTarget.Repo - ); - - /** @type {EndpointConfigData} */ - const endpointConfig = ( - await gitRepo.getContents( - gitHubTarget.Branch, - gitHubTarget.ConfigPath, - true - ) - ).data.data; - - return endpointConfig; -}; - /** * returns true if there are any items that match in both arrays * @@ -93,12 +60,12 @@ const anythingInArrayMatch = (array1, array2) => /** * Addts a CommitResult to the Report if it exists * - * @param {CommitReport[]} Report - * @param {CommitReport} [CommitResult] + * @param {GithubCompare[]} Report + * @param {GitHubTreePush} Tree */ -const addToReport = (Report, CommitResult) => { - if (CommitResult) { - Report.push(CommitResult); +const addToReport = (Report, Tree) => { + if (Tree.lastCompare) { + Report.push(Tree.lastCompare); } }; @@ -116,17 +83,30 @@ const SyncEndpoint = async ( gitHubCredentials, gitHubCommitter ) => { - /** @type {CommitReport[]} */ + /** @type {GithubCompare[]} */ const report = []; - const gitModule = new GitHub(gitHubCredentials); - // @ts-ignore - const gitRepo = await gitModule.getRepo( - gitHubTarget.Owner, - gitHubTarget.Repo + const configTreeConfig = { + owner: gitHubTarget.Owner, + base: gitHubTarget.Branch, + repo: gitHubTarget.Repo + }; + + const configTree = new GitHubTreePush( + gitHubCredentials.token, + configTreeConfig ); - const endpointConfig = await getRemoteConfig(gitHubTarget, gitHubCredentials); + //https://docs.github.com/en/rest/reference/repos#contents + const endpointConfigResponse = await configTree.__fetchResponse( + `/contents/${gitHubTarget.ConfigPath}?ref=${configTreeConfig.base}`, + configTree.__gitDefaultOptions({ + headers: { Accept: "application/vnd.github.v3.raw" } + }) + ); + + /** @type {EndpointConfigData} */ + const endpointConfig = (await endpointConfigResponse.json()).data; const wordPressApiUrl = sourceEndpointConfig.WordPressSource.url + apiPath; const allApiRequests = @@ -176,8 +156,11 @@ const SyncEndpoint = async ( return; } - const repoDetails = await gitRepo.getDetails(); - if (!repoDetails.data.permissions.push) { + //https://docs.github.com/en/rest/reference/repos#get-a-repository + + const repoDetails = await configTree.__getSomeJson(""); + + if (!repoDetails.permissions.push) { throw new Error( `App user has no write permissions for ${gitHubTarget.Repo}` ); @@ -232,25 +215,16 @@ const SyncEndpoint = async ( .join("/"); const fileName = endpointConfig.GeneralFilePath.split("/").slice(-1)[0]; - const fileMap = new Map(); - fileMap.set(fileName, jsonData); - const newTree = await createTreeFromFileMap( - gitRepo, - gitHubTarget.Branch, - fileMap, - filePath - ); + const generalTree = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path: filePath, + commit_message: commitTitleGeneral + }); - addToReport( - report, - await CommitIfChanged( - gitRepo, - gitHubTarget.Branch, - newTree, - commitTitleGeneral, - gitHubCommitter - ) - ); + generalTree.syncFile(fileName, JSON.stringify(jsonData, null, 2)); + await generalTree.treePush(); + + addToReport(report, generalTree); } /** @type {Map | null} */ diff --git a/wordpress-to-github/package.json b/wordpress-to-github/package.json index 28d2a5e..ebff227 100644 --- a/wordpress-to-github/package.json +++ b/wordpress-to-github/package.json @@ -29,8 +29,7 @@ }, "dependencies": { "fetch-retry": "^4.1.1", - "github-api": "^3.4.0", "node-fetch": "^2.6.1", - "sha1": "^1.1.1" + "@cagov/github-tree-push": "^0.0.1" } } From 4cd75b92c7b15375dd6a17d5996b39a38f4555c7 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 11:33:09 -0800 Subject: [PATCH 02/18] Work in progress tree map --- wordpress-to-github/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 1b402c0..fe0e64a 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -221,7 +221,7 @@ const SyncEndpoint = async ( commit_message: commitTitleGeneral }); - generalTree.syncFile(fileName, JSON.stringify(jsonData, null, 2)); + generalTree.syncFile(fileName, jsonData); await generalTree.treePush(); addToReport(report, generalTree); From ea85e1fa575ff20590e80e9c93e2f1d3bbba9cd9 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 14:42:10 -0800 Subject: [PATCH 03/18] adding links meta --- wordpress-to-github/common/index.js | 1 + wordpress-to-github/index.js | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/wordpress-to-github/common/index.js b/wordpress-to-github/common/index.js index 4948e4f..c591ab2 100644 --- a/wordpress-to-github/common/index.js +++ b/wordpress-to-github/common/index.js @@ -138,6 +138,7 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { * @property {number} [featured_media] * @property {{}[]} [media] * @property {WordpressMediaSize[]} [sizes] + * @property {{self:{href:string}[]}} [_links] */ /** diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index fe0e64a..45e381b 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -249,8 +249,7 @@ const SyncEndpoint = async ( ) }; - /** @type {string} */ - const object_url = jsonData["_links"]?.self[0].href; + const object_url = jsonData._links?.self[0].href; removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); @@ -399,8 +398,7 @@ const SyncEndpoint = async ( addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - /** @type {string} */ - const object_url = jsonData["_links"]?.self[0].href; + const object_url = jsonData._links?.self[0].href; removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); @@ -453,8 +451,7 @@ const SyncEndpoint = async ( addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - /** @type {string} */ - const object_url = jsonData["_links"]?.self[0].href; + const object_url = jsonData._links?.self[0].href; removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); From ed543884e5732cea4045159a34aad49a42f72621 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 15:07:11 -0800 Subject: [PATCH 04/18] Work in progress on media --- wordpress-to-github/index.js | 55 ++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 45e381b..e30391a 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -235,9 +235,13 @@ const SyncEndpoint = async ( const mediaMap = endpointConfig.MediaPath ? new Map() : null; // MEDIA - const mediaContentPlaceholder = - "TBD : Binary file to be updated in a later step"; if (endpointConfig.MediaPath && mediaMap && allMedia) { + const mediaTree2 = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path: endpointConfig.MediaPath, + commit_message: commitTitleMedia + }); + allMedia.forEach(x => { /** @type {GithubOutputJson} */ const jsonData = { @@ -268,38 +272,39 @@ const SyncEndpoint = async ( //put binary placeholders so they aren't deleted. Will search for these if an update happens. for (const s of jsonData.sizes) { - mediaMap.set(s.path, mediaContentPlaceholder); + mediaTree2.doNotRemoveFile(s.path); } } //PDF jsonData.path = pathFromMediaSourceUrl(x.source_url); - mediaMap.set(jsonData.path, mediaContentPlaceholder); - mediaMap.set( - pathFromMediaSourceUrl(x.source_url).replace(/\.([^.]+)$/, ".json"), - wrapInFileMeta( - sourceEndpointConfig.WordPressSource.url, - gitHubTarget, - fieldMetaReference.media, - jsonData, - object_url - ) + mediaTree2.doNotRemoveFile(jsonData.path); + + const mediaJson = wrapInFileMeta( + sourceEndpointConfig.WordPressSource.url, + gitHubTarget, + fieldMetaReference.media, + jsonData, + object_url + ); + + const mediaPath = pathFromMediaSourceUrl(x.source_url).replace( + /\.([^.]+)$/, + ".json" ); + + mediaTree2.syncFile(mediaPath, mediaJson); + + mediaMap.set(mediaPath, mediaJson); }); - let mediaTree = await createTreeFromFileMap( - gitRepo, - gitHubTarget.Branch, - mediaMap, - endpointConfig.MediaPath, - true - ); + //TODO: Need to figure out which meta files changed... - const mediaChanges = mediaTree - .filter(x => x.content && x.content !== mediaContentPlaceholder) - .map(mt => JSON.parse(mt.content).data); + const mediaChanges = (await mediaTree2.treePushDryRun()).map(p => + mediaMap.get(p) + ); if (mediaChanges.length) { - console.log(`Checking ${mediaTree.length} media items`); + console.log(`Checking ${mediaChanges.length} media items`); /** @type {Promise[]} */ const binarySyncs = []; @@ -309,6 +314,8 @@ const SyncEndpoint = async ( if (mediaTreeItem.sizes) { //Sized images for (const sizeJson of mediaTreeItem.sizes) { + mediaTree2.syncFile(mediaPath, mediaJson); + binarySyncs.push( syncBinaryFile( sizeJson.wordpress_url, From 57fec93b813764b2650fe135492d1ac30c5de313 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 15:11:07 -0800 Subject: [PATCH 05/18] path wip --- wordpress-to-github/index.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index e30391a..8a6b5d8 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -40,6 +40,10 @@ const fieldMetaReference = { /** @type {Map } */ const updateCache = new Map(); const cacheObjects = ["media", "posts", "pages"]; +const fetch = require("fetch-retry")(require("node-fetch/lib"), { + retries: 3, + retryDelay: 2000 +}); /** * returns true if there are any items that match in both arrays @@ -314,6 +318,13 @@ const SyncEndpoint = async ( if (mediaTreeItem.sizes) { //Sized images for (const sizeJson of mediaTreeItem.sizes) { + const wordpress_url = sizeJson.wordpress_url; + + console.log(`Downloading...${wordpress_url}`); + const fetchResponse = await fetch(wordpress_url); + const blob = await fetchResponse.arrayBuffer(); + const buffer = Buffer.from(blob); + mediaTree2.syncFile(mediaPath, mediaJson); binarySyncs.push( From 0731e0ad5a976af709b132ca3be08b1579129596 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 15:55:01 -0800 Subject: [PATCH 06/18] media tree done i think --- wordpress-to-github/index.js | 55 +++++++++++++----------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 8a6b5d8..ca88e57 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -240,7 +240,7 @@ const SyncEndpoint = async ( // MEDIA if (endpointConfig.MediaPath && mediaMap && allMedia) { - const mediaTree2 = new GitHubTreePush(gitHubCredentials.token, { + const mediaTree = new GitHubTreePush(gitHubCredentials.token, { ...configTreeConfig, path: endpointConfig.MediaPath, commit_message: commitTitleMedia @@ -276,12 +276,12 @@ const SyncEndpoint = async ( //put binary placeholders so they aren't deleted. Will search for these if an update happens. for (const s of jsonData.sizes) { - mediaTree2.doNotRemoveFile(s.path); + mediaTree.doNotRemoveFile(s.path); } } //PDF jsonData.path = pathFromMediaSourceUrl(x.source_url); - mediaTree2.doNotRemoveFile(jsonData.path); + mediaTree.doNotRemoveFile(jsonData.path); const mediaJson = wrapInFileMeta( sourceEndpointConfig.WordPressSource.url, @@ -296,15 +296,15 @@ const SyncEndpoint = async ( ".json" ); - mediaTree2.syncFile(mediaPath, mediaJson); + mediaTree.syncFile(mediaPath, mediaJson); mediaMap.set(mediaPath, mediaJson); }); //TODO: Need to figure out which meta files changed... - const mediaChanges = (await mediaTree2.treePushDryRun()).map(p => - mediaMap.get(p) + const mediaChanges = (await mediaTree.treePushDryRun()).map( + p => mediaMap.get(p.replace(`${endpointConfig.MediaPath}/`, "")).data ); if (mediaChanges.length) { @@ -325,45 +325,30 @@ const SyncEndpoint = async ( const blob = await fetchResponse.arrayBuffer(); const buffer = Buffer.from(blob); - mediaTree2.syncFile(mediaPath, mediaJson); + const path = pathFromMediaSourceUrl(wordpress_url); - binarySyncs.push( - syncBinaryFile( - sizeJson.wordpress_url, - gitRepo, - mediaTree, - endpointConfig - ) - ); + mediaTree.syncFile(path, buffer); } } //not sized media (PDF or non-image) - binarySyncs.push( - syncBinaryFile( - mediaTreeItem.wordpress_url, - gitRepo, - mediaTree, - endpointConfig - ) - ); + const wordpress_url = mediaTreeItem.wordpress_url; + + console.log(`Downloading...${wordpress_url}`); + const fetchResponse = await fetch(wordpress_url); + const blob = await fetchResponse.arrayBuffer(); + const buffer = Buffer.from(blob); + + const path = pathFromMediaSourceUrl(wordpress_url); + + mediaTree.syncFile(path, buffer); } await Promise.all(binarySyncs); } - //Remove any leftover binary placeholders... - mediaTree = mediaTree.filter(x => x.content !== mediaContentPlaceholder); - addToReport( - report, - await CommitIfChanged( - gitRepo, - gitHubTarget.Branch, - mediaTree, - `${commitTitleMedia} (${mediaTree.length} updates)`, - gitHubCommitter - ) - ); + await mediaTree.treePush(); + addToReport(report, mediaTree); } /** From b49eb22dcb071b0c11ac25840c225f1584c42979 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Wed, 8 Dec 2021 15:57:01 -0800 Subject: [PATCH 07/18] media syncs working --- wordpress-to-github/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index ca88e57..8e68f9a 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -311,7 +311,7 @@ const SyncEndpoint = async ( console.log(`Checking ${mediaChanges.length} media items`); /** @type {Promise[]} */ - const binarySyncs = []; + const binarySyncs = []; //TODO get download syncs working again. //Pull in binaries for any media meta changes for (const mediaTreeItem of mediaChanges) { From eef4a6c9935a4e44521df6969ad74b5be5075e73 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 10:31:44 -0800 Subject: [PATCH 08/18] Posts update --- wordpress-to-github/index.js | 86 +++++++++++++++++------------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 8e68f9a..eced180 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -231,8 +231,6 @@ const SyncEndpoint = async ( addToReport(report, generalTree); } - /** @type {Map | null} */ - const postMap = endpointConfig.PostPath ? new Map() : null; /** @type {Map | null} */ const pagesMap = endpointConfig.PagePath ? new Map() : null; /** @type {Map | null} */ @@ -243,7 +241,8 @@ const SyncEndpoint = async ( const mediaTree = new GitHubTreePush(gitHubCredentials.token, { ...configTreeConfig, path: endpointConfig.MediaPath, - commit_message: commitTitleMedia + commit_message: commitTitleMedia, + removeOtherFiles: true }); allMedia.forEach(x => { @@ -393,57 +392,52 @@ const SyncEndpoint = async ( }; // POSTS - if (endpointConfig.PostPath && postMap && allPosts) { - allPosts.forEach(x => { - const jsonData = wordPressRowToGitHubOutput(x); + if (endpointConfig.PostPath && allPosts) { + const postTree = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path: endpointConfig.PostPath, + commit_message: commitTitlePosts, + removeOtherFiles: true + }); - const HTML = cleanupContent(x.content); + allPosts.forEach(wordPressRow => { + const jsonData = wordPressRowToGitHubOutput(wordPressRow); - addMediaSection(endpointConfig, mediaMap, jsonData, HTML); + if ( + anythingInArrayMatch( + jsonData.tags, + sourceEndpointConfig.WordPressSource.tags_exclude + ) + ) { + //ignoring this file from tags_exclude - const object_url = jsonData._links?.self[0].href; + postTree.doNotRemoveFile(`${wordPressRow.slug}.json`); + postTree.doNotRemoveFile(`${wordPressRow.slug}.html`); + } else { + const HTML = cleanupContent(wordPressRow.content); - removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - const ignoreThisOne = anythingInArrayMatch( - jsonData.tags, - sourceEndpointConfig.WordPressSource.tags_exclude - ); + const object_url = jsonData._links?.self[0].href; - postMap.set( - `${x.slug}.json`, - ignoreThisOne - ? null - : wrapInFileMeta( - sourceEndpointConfig.WordPressSource.url, - gitHubTarget, - fieldMetaReference.posts, - jsonData, - object_url - ) - ); - postMap.set(`${x.slug}.html`, ignoreThisOne ? null : HTML); + removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + + postTree.syncFile( + `${wordPressRow.slug}.json`, + wrapInFileMeta( + sourceEndpointConfig.WordPressSource.url, + gitHubTarget, + fieldMetaReference.posts, + jsonData, + object_url + ) + ); + postTree.syncFile(`${wordPressRow.slug}.html`, HTML); + } }); - const postTree = await createTreeFromFileMap( - gitRepo, - gitHubTarget.Branch, - postMap, - endpointConfig.PostPath, - true - ); - addToReport( - report, - await CommitIfChanged( - gitRepo, - gitHubTarget.Branch, - postTree, - `${commitTitlePosts} (${ - postTree.filter(x => x.path.endsWith(".html")).length - } updates)`, - gitHubCommitter - ) - ); + await postTree.treePush(); + addToReport(report, postTree); } // PAGES if (endpointConfig.PagePath && pagesMap && allPages) { From 90471dcd3f1899d535e65486e423f3552ca338f9 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 10:38:39 -0800 Subject: [PATCH 09/18] pages tree --- wordpress-to-github/index.js | 82 ++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 45 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index eced180..7f0163e 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -231,8 +231,6 @@ const SyncEndpoint = async ( addToReport(report, generalTree); } - /** @type {Map | null} */ - const pagesMap = endpointConfig.PagePath ? new Map() : null; /** @type {Map | null} */ const mediaMap = endpointConfig.MediaPath ? new Map() : null; @@ -440,57 +438,51 @@ const SyncEndpoint = async ( addToReport(report, postTree); } // PAGES - if (endpointConfig.PagePath && pagesMap && allPages) { - allPages.forEach(x => { - const jsonData = wordPressRowToGitHubOutput(x); + if (endpointConfig.PagePath && allPages) { + const pagesTree = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path: endpointConfig.PagePath, + commit_message: commitTitlePages, + removeOtherFiles: true + }); - const HTML = cleanupContent(x.content); + allPages.forEach(wordPressRow => { + const jsonData = wordPressRowToGitHubOutput(wordPressRow); + if ( + anythingInArrayMatch( + jsonData.tags, + sourceEndpointConfig.WordPressSource.tags_exclude + ) + ) { + //ignoring this file from tags_exclude - addMediaSection(endpointConfig, mediaMap, jsonData, HTML); + pagesTree.doNotRemoveFile(`${wordPressRow.slug}.json`); + pagesTree.doNotRemoveFile(`${wordPressRow.slug}.html`); + } else { + const HTML = cleanupContent(wordPressRow.content); - const object_url = jsonData._links?.self[0].href; + addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + const object_url = jsonData._links?.self[0].href; - const ignoreThisOne = anythingInArrayMatch( - jsonData.tags, - sourceEndpointConfig.WordPressSource.tags_exclude - ); + removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); - pagesMap.set( - `${x.slug}.json`, - ignoreThisOne - ? null - : wrapInFileMeta( - sourceEndpointConfig.WordPressSource.url, - gitHubTarget, - fieldMetaReference.media, - jsonData, - object_url - ) - ); - pagesMap.set(`${x.slug}.html`, ignoreThisOne ? null : HTML); + pagesTree.syncFile( + `${wordPressRow.slug}.json`, + wrapInFileMeta( + sourceEndpointConfig.WordPressSource.url, + gitHubTarget, + fieldMetaReference.pages, + jsonData, + object_url + ) + ); + pagesTree.syncFile(`${wordPressRow.slug}.html`, HTML); + } }); - const pagesTree = await createTreeFromFileMap( - gitRepo, - gitHubTarget.Branch, - pagesMap, - endpointConfig.PagePath, - true - ); - addToReport( - report, - await CommitIfChanged( - gitRepo, - gitHubTarget.Branch, - pagesTree, - `${commitTitlePages} (${ - pagesTree.filter(x => x.path.endsWith(".html")).length - } updates)`, //TODO: Pull from a name property - gitHubCommitter - ) - ); + await pagesTree.treePush(); + addToReport(report, pagesTree); } // API Requests From 3e07fa0f62e0bce52580dc20b9a502df9b2ca0f9 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 10:51:16 -0800 Subject: [PATCH 10/18] fully working --- wordpress-to-github/index.js | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 7f0163e..ebd1b03 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -503,29 +503,29 @@ const SyncEndpoint = async ( // Create and commit a git tree for each set of files. for (let [folderName, fileMap] of Object.entries(apiRequestsByFolder)) { - const requestsTree = await createTreeFromFileMap( - gitRepo, - gitHubTarget.Branch, - fileMap, - folderName, - false - ); - + const requestsTree = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path: folderName, + commit_message: commitTitlePages, + removeOtherFiles: false + }); + + /** @type {Map} */ + const mapList = fileMap; + mapList.forEach((value, path) => { + requestsTree.syncFile(path, value); + }); + + await requestsTree.treePush(); + + /* const reportLabel = folderName.split("/").slice(-1).join("/") || "root"; const updateCount = `${requestsTree.length} ${ requestsTree.length === 1 ? "update" : "updates" }`; + */ - addToReport( - report, - await CommitIfChanged( - gitRepo, - gitHubTarget.Branch, - requestsTree, - `${commitTitleApiRequests} (${updateCount} to ${reportLabel})`, - gitHubCommitter - ) - ); + addToReport(report, requestsTree); } } From d4f44b1d9fe78544a26fc73d5d20ae7989978700 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 11:39:12 -0800 Subject: [PATCH 11/18] Compare reporting --- WordpressSync/index.js | 29 +++++++++++------------------ wordpress-to-github/common/index.js | 1 - wordpress-to-github/index.js | 7 ++----- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/WordpressSync/index.js b/WordpressSync/index.js index 3bb47b3..d7ac0f3 100644 --- a/WordpressSync/index.js +++ b/WordpressSync/index.js @@ -1,7 +1,6 @@ // @ts-check const { SyncEndpoint } = require("@cagov/wordpress-to-github"); const { - GitHubCommitter, GitHubCredentials, SourceEndpointConfigData } = require("@cagov/wordpress-to-github/common"); @@ -11,11 +10,6 @@ const debugChannel = "C01DBP67MSQ"; // #testingbot const endPointsJson = require("./endpoints.json"); /** @type {SourceEndpointConfigData[]} */ const endpoints = endPointsJson.data.projects; -/** @type {GitHubCommitter} **/ -const gitHubCommitter = { - name: `${process.env["GITHUB_NAME"]}`, - email: `${process.env["GITHUB_EMAIL"]}` -}; /** @type {GitHubCredentials} **/ const gitHubCredentials = { token: `${process.env["GITHUB_TOKEN"]}` @@ -84,8 +78,7 @@ const doProcessEndpoints = async work => { const commitReports = await SyncEndpoint( endpoint.GitHubTarget, endpoint, - gitHubCredentials, - gitHubCommitter + gitHubCredentials ); if (endpoint.ReportingChannel_Slack && slackBotGetToken()) { @@ -95,11 +88,11 @@ const doProcessEndpoints = async work => { /** @type {string[]} */ let mergeFileNames = []; - commitReports.forEach(x => { - opCount += x.Files.length; + commitReports.forEach(compare => { + opCount += compare.files.length; mergeFileNames.push( - ...x.Files.map( + ...compare.files.map( //Remove file extension, and remove resolution postfix f => f.filename @@ -124,16 +117,16 @@ const doProcessEndpoints = async work => { ); for (const commitReport of commitReports) { - const fileData = commitReport.Files.map( - x => `• ${x.status} - _${x.filename.split("/").slice(-1)[0]}_` - ).join("\n"); + const fileData = commitReport.files + .map(x => `• ${x.status} - _${x.filename.split("/").slice(-1)[0]}_`) + .join("\n"); await slackBot.Reply( - `<${commitReport.Commit.html_url}|${ - commitReport.Commit.message + `<${commitReport.commit.html_url}|${ + commitReport.commit.message }>\n${ - commitReport.Files.length > 1 - ? `${commitReport.Files.length} changes\n` + commitReport.files.length > 1 + ? `${commitReport.files.length} changes\n` : "" }${fileData}` ); diff --git a/wordpress-to-github/common/index.js b/wordpress-to-github/common/index.js index c591ab2..6f60c83 100644 --- a/wordpress-to-github/common/index.js +++ b/wordpress-to-github/common/index.js @@ -39,7 +39,6 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { /** * @typedef {{Owner:string, Repo:string, Branch:string, ConfigPath:string}} GitHubTarget - * @typedef {{name:string, email:string}} GitHubCommitter * @typedef {{token:string}} GitHubCredentials * @typedef {{url:string,tags_exclude:string[]}} WordpressSource * @typedef {{width:number,path:string}} WordpressMediaSize diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index ebd1b03..8f13eb7 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -21,7 +21,6 @@ const { SourceEndpointConfigData, WordpressApiDateCacheItem, WordpressApiHashCacheItem, - GitHubCommitter, GithubOutputJson, WordpressMediaRow, WordpressPageRow, @@ -79,13 +78,11 @@ const addToReport = (Report, Tree) => { * @param {GitHubTarget} gitHubTarget * @param {SourceEndpointConfigData} sourceEndpointConfig * @param {GitHubCredentials} gitHubCredentials - * @param {GitHubCommitter} gitHubCommitter */ const SyncEndpoint = async ( gitHubTarget, sourceEndpointConfig, - gitHubCredentials, - gitHubCommitter + gitHubCredentials ) => { /** @type {GithubCompare[]} */ const report = []; @@ -506,7 +503,7 @@ const SyncEndpoint = async ( const requestsTree = new GitHubTreePush(gitHubCredentials.token, { ...configTreeConfig, path: folderName, - commit_message: commitTitlePages, + commit_message: commitTitleApiRequests, removeOtherFiles: false }); From cb6a9a73ba2321f584f87ce438ce895da1b6aba0 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 13:08:30 -0800 Subject: [PATCH 12/18] code combine and media ref cleanup --- wordpress-to-github/index.js | 168 ++++++++++++++++------------------- 1 file changed, 76 insertions(+), 92 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 8f13eb7..768876c 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -34,7 +34,7 @@ const commitTitleGeneral = "Wordpress General File Update"; const fieldMetaReference = { posts: "https://developer.wordpress.org/rest-api/reference/posts/", pages: "https://developer.wordpress.org/rest-api/reference/pages/", - media: "https://developer.wordpress.org/rest-api/reference/pages/" + media: "https://developer.wordpress.org/rest-api/reference/media/" }; /** @type {Map } */ const updateCache = new Map(); @@ -295,8 +295,7 @@ const SyncEndpoint = async ( mediaMap.set(mediaPath, mediaJson); }); - //TODO: Need to figure out which meta files changed... - + //Figure out which meta files changed... const mediaChanges = (await mediaTree.treePushDryRun()).map( p => mediaMap.get(p.replace(`${endpointConfig.MediaPath}/`, "")).data ); @@ -386,101 +385,86 @@ const SyncEndpoint = async ( return jsonData; }; - // POSTS - if (endpointConfig.PostPath && allPosts) { - const postTree = new GitHubTreePush(gitHubCredentials.token, { - ...configTreeConfig, - path: endpointConfig.PostPath, - commit_message: commitTitlePosts, - removeOtherFiles: true - }); - - allPosts.forEach(wordPressRow => { - const jsonData = wordPressRowToGitHubOutput(wordPressRow); - - if ( - anythingInArrayMatch( - jsonData.tags, - sourceEndpointConfig.WordPressSource.tags_exclude - ) - ) { - //ignoring this file from tags_exclude - - postTree.doNotRemoveFile(`${wordPressRow.slug}.json`); - postTree.doNotRemoveFile(`${wordPressRow.slug}.html`); - } else { - const HTML = cleanupContent(wordPressRow.content); - - addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - - const object_url = jsonData._links?.self[0].href; + /** + * Common Tree sync operations for Pages or Posts + * + * @param {string} path + * @param {*[]} WordPressRows + * @param {string} commit_message + * @param {string} fieldMetaRefUrl + */ + const treeWorkForPagesOrPosts = async ( + path, + WordPressRows, + commit_message, + fieldMetaRefUrl + ) => { + if (path && WordPressRows?.length) { + const objectTree = new GitHubTreePush(gitHubCredentials.token, { + ...configTreeConfig, + path, + commit_message, + removeOtherFiles: true + }); - removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + WordPressRows.forEach(wordPressRow => { + const jsonData = wordPressRowToGitHubOutput(wordPressRow); + const slug = wordPressRow.slug; + const fileNameJson = `${slug}.json`; + const fileNameHtml = `${slug}.html`; - postTree.syncFile( - `${wordPressRow.slug}.json`, - wrapInFileMeta( - sourceEndpointConfig.WordPressSource.url, - gitHubTarget, - fieldMetaReference.posts, - jsonData, - object_url + if ( + anythingInArrayMatch( + jsonData.tags, + sourceEndpointConfig.WordPressSource.tags_exclude ) - ); - postTree.syncFile(`${wordPressRow.slug}.html`, HTML); - } - }); - - await postTree.treePush(); - addToReport(report, postTree); - } - // PAGES - if (endpointConfig.PagePath && allPages) { - const pagesTree = new GitHubTreePush(gitHubCredentials.token, { - ...configTreeConfig, - path: endpointConfig.PagePath, - commit_message: commitTitlePages, - removeOtherFiles: true - }); - - allPages.forEach(wordPressRow => { - const jsonData = wordPressRowToGitHubOutput(wordPressRow); - if ( - anythingInArrayMatch( - jsonData.tags, - sourceEndpointConfig.WordPressSource.tags_exclude - ) - ) { - //ignoring this file from tags_exclude - - pagesTree.doNotRemoveFile(`${wordPressRow.slug}.json`); - pagesTree.doNotRemoveFile(`${wordPressRow.slug}.html`); - } else { - const HTML = cleanupContent(wordPressRow.content); - - addMediaSection(endpointConfig, mediaMap, jsonData, HTML); - - const object_url = jsonData._links?.self[0].href; + ) { + //ignoring this file from tags_exclude + + objectTree.doNotRemoveFile(fileNameJson); + objectTree.doNotRemoveFile(fileNameHtml); + } else { + const HTML = cleanupContent(wordPressRow.content); + const object_url = jsonData._links?.self[0].href; + + addMediaSection(endpointConfig, mediaMap, jsonData, HTML); + + removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + + objectTree.syncFile( + fileNameJson, + wrapInFileMeta( + sourceEndpointConfig.WordPressSource.url, + gitHubTarget, + fieldMetaRefUrl, + jsonData, + object_url + ) + ); + objectTree.syncFile(fileNameHtml, HTML); + } + }); - removeExcludedProperties(jsonData, endpointConfig.ExcludeProperties); + await objectTree.treePush(); + addToReport(report, objectTree); + } + }; - pagesTree.syncFile( - `${wordPressRow.slug}.json`, - wrapInFileMeta( - sourceEndpointConfig.WordPressSource.url, - gitHubTarget, - fieldMetaReference.pages, - jsonData, - object_url - ) - ); - pagesTree.syncFile(`${wordPressRow.slug}.html`, HTML); - } - }); + // POSTS + await treeWorkForPagesOrPosts( + endpointConfig.PostPath, + allPosts, + commitTitlePosts, + fieldMetaReference.posts + ); - await pagesTree.treePush(); - addToReport(report, pagesTree); - } + // PAGES + await treeWorkForPagesOrPosts( + endpointConfig.PagePath, + allPages, + commitTitlePages, + fieldMetaReference.pages + ); // API Requests if (allApiRequests) { From da251fc239c8c0e8b97d0dab6ef0c8efa5fa427f Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 13:37:03 -0800 Subject: [PATCH 13/18] using media download promise --- wordpress-to-github/index.js | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 768876c..518001c 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -39,10 +39,6 @@ const fieldMetaReference = { /** @type {Map } */ const updateCache = new Map(); const cacheObjects = ["media", "posts", "pages"]; -const fetch = require("fetch-retry")(require("node-fetch/lib"), { - retries: 3, - retryDelay: 2000 -}); /** * returns true if there are any items that match in both arrays @@ -303,9 +299,6 @@ const SyncEndpoint = async ( if (mediaChanges.length) { console.log(`Checking ${mediaChanges.length} media items`); - /** @type {Promise[]} */ - const binarySyncs = []; //TODO get download syncs working again. - //Pull in binaries for any media meta changes for (const mediaTreeItem of mediaChanges) { if (mediaTreeItem.sizes) { @@ -313,31 +306,19 @@ const SyncEndpoint = async ( for (const sizeJson of mediaTreeItem.sizes) { const wordpress_url = sizeJson.wordpress_url; - console.log(`Downloading...${wordpress_url}`); - const fetchResponse = await fetch(wordpress_url); - const blob = await fetchResponse.arrayBuffer(); - const buffer = Buffer.from(blob); - const path = pathFromMediaSourceUrl(wordpress_url); - mediaTree.syncFile(path, buffer); + mediaTree.syncDownload(path, wordpress_url); } } //not sized media (PDF or non-image) const wordpress_url = mediaTreeItem.wordpress_url; - console.log(`Downloading...${wordpress_url}`); - const fetchResponse = await fetch(wordpress_url); - const blob = await fetchResponse.arrayBuffer(); - const buffer = Buffer.from(blob); - const path = pathFromMediaSourceUrl(wordpress_url); - mediaTree.syncFile(path, buffer); + mediaTree.syncDownload(path, wordpress_url); } - - await Promise.all(binarySyncs); } await mediaTree.treePush(); From a4ae751222b12afe90193711de556268e9d630f8 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 14:32:21 -0800 Subject: [PATCH 14/18] cleaned up reporting --- WordpressSync/index.js | 50 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/WordpressSync/index.js b/WordpressSync/index.js index d7ac0f3..92a71d2 100644 --- a/WordpressSync/index.js +++ b/WordpressSync/index.js @@ -80,29 +80,33 @@ const doProcessEndpoints = async work => { endpoint, gitHubCredentials ); + if (commitReports?.length) { + //Commits happened + + let opCount = 0; + + /** @type {string[]} */ + let mergeFileNames = []; + commitReports.forEach(compare => { + opCount += compare.files.length; + + mergeFileNames.push( + ...compare.files.map( + //Remove file extension, and remove resolution postfix + f => + f.filename + .split("/") + .slice(-1)[0] + .split(".")[0] + .replace(/-\d{1,4}x\d{1,4}$/, "") + ) + ); + }); - if (endpoint.ReportingChannel_Slack && slackBotGetToken()) { - //Endpoint reporting channel enabled. Add a post for each commit report. - if (commitReports?.length) { - let opCount = 0; - - /** @type {string[]} */ - let mergeFileNames = []; - commitReports.forEach(compare => { - opCount += compare.files.length; - - mergeFileNames.push( - ...compare.files.map( - //Remove file extension, and remove resolution postfix - f => - f.filename - .split("/") - .slice(-1)[0] - .split(".")[0] - .replace(/-\d{1,4}x\d{1,4}$/, "") - ) - ); - }); + const slugList = ` : _${[...new Set(mergeFileNames)].join(", ")}_`; + + if (endpoint.ReportingChannel_Slack && slackBotGetToken()) { + //Endpoint reporting channel enabled. Add a post for each commit report. const slackBot = new SlackBot( slackBotGetToken(), @@ -110,8 +114,6 @@ const doProcessEndpoints = async work => { { username: endpoint.name } ); - const slugList = ` : _${[...new Set(mergeFileNames)].join(", ")}_`; - await slackBot.Chat( `${opCount} changes${slugList.length < 300 ? slugList : "."}` ); From f627396472df3bb48fc0160e43fd68d75ae3d571 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 17:05:08 -0800 Subject: [PATCH 15/18] github tree push v update --- package-lock.json | 16 ++++++++-------- wordpress-to-github/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index fbba292..df07734 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,9 +21,9 @@ } }, "node_modules/@cagov/github-tree-push": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.0.1.tgz", - "integrity": "sha512-JPrb48dO8n2gkIKtdn0jFxY7oSUS72y5Udvy/0ILMR28NIxTwhcYGsv9L33hBc3YLIPuB0Tw/GoX6dzObBYttw==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.1.0.tgz", + "integrity": "sha512-8QChOMflnIvmXZgpMXhGwb9wURLhveuwGuDP1iWreKyPnF1eoz+Amg2Sj7JuRPnOdvoFurr4Ln8tAEG2uIrQ8g==", "dependencies": { "fetch-retry": "^5.0.1", "node-fetch": "^2.6.6", @@ -1233,7 +1233,7 @@ "version": "0.2.0", "license": "MIT", "dependencies": { - "@cagov/github-tree-push": "^0.0.1", + "@cagov/github-tree-push": "^0.1.0", "fetch-retry": "^4.1.1", "node-fetch": "^2.6.1" } @@ -1241,9 +1241,9 @@ }, "dependencies": { "@cagov/github-tree-push": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.0.1.tgz", - "integrity": "sha512-JPrb48dO8n2gkIKtdn0jFxY7oSUS72y5Udvy/0ILMR28NIxTwhcYGsv9L33hBc3YLIPuB0Tw/GoX6dzObBYttw==", + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@cagov/github-tree-push/-/github-tree-push-0.1.0.tgz", + "integrity": "sha512-8QChOMflnIvmXZgpMXhGwb9wURLhveuwGuDP1iWreKyPnF1eoz+Amg2Sj7JuRPnOdvoFurr4Ln8tAEG2uIrQ8g==", "requires": { "fetch-retry": "^5.0.1", "node-fetch": "^2.6.6", @@ -1269,7 +1269,7 @@ "@cagov/wordpress-to-github": { "version": "file:wordpress-to-github", "requires": { - "@cagov/github-tree-push": "^0.0.1", + "@cagov/github-tree-push": "^0.1.0", "fetch-retry": "^4.1.1", "node-fetch": "^2.6.1" } diff --git a/wordpress-to-github/package.json b/wordpress-to-github/package.json index ebff227..4edc764 100644 --- a/wordpress-to-github/package.json +++ b/wordpress-to-github/package.json @@ -30,6 +30,6 @@ "dependencies": { "fetch-retry": "^4.1.1", "node-fetch": "^2.6.1", - "@cagov/github-tree-push": "^0.0.1" + "@cagov/github-tree-push": "^0.1.0" } } From f0d61a0c3347a2d85e6d9011363fa8be4ecf4b47 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 17:07:00 -0800 Subject: [PATCH 16/18] delta reduce --- WordpressSync/endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WordpressSync/endpoints.json b/WordpressSync/endpoints.json index 239ee1c..85c1bc0 100644 --- a/WordpressSync/endpoints.json +++ b/WordpressSync/endpoints.json @@ -10,7 +10,7 @@ "name": "Covid 19 website", "description": "Using data to create test output.", "enabled": false, - "enabledLocal": true, + "enabledLocal": false, "ReportingChannel_Slack": "C01H6RB99E2", "WordPressSource": { "url": "https://as-go-covid19-d-001.azurewebsites.net", From 8ee131ce12f55ea9c0746827235713f154b23ae0 Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 17:31:02 -0800 Subject: [PATCH 17/18] simlify token --- WordpressSync/index.js | 9 +++------ wordpress-to-github/common/index.js | 1 - wordpress-to-github/index.js | 18 +++++++----------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/WordpressSync/index.js b/WordpressSync/index.js index 92a71d2..ad74e14 100644 --- a/WordpressSync/index.js +++ b/WordpressSync/index.js @@ -1,7 +1,6 @@ // @ts-check const { SyncEndpoint } = require("@cagov/wordpress-to-github"); const { - GitHubCredentials, SourceEndpointConfigData } = require("@cagov/wordpress-to-github/common"); const SlackBot = require("@cagov/slack-connector"); @@ -10,10 +9,8 @@ const debugChannel = "C01DBP67MSQ"; // #testingbot const endPointsJson = require("./endpoints.json"); /** @type {SourceEndpointConfigData[]} */ const endpoints = endPointsJson.data.projects; -/** @type {GitHubCredentials} **/ -const gitHubCredentials = { - token: `${process.env["GITHUB_TOKEN"]}` -}; + +const gitHubToken = `${process.env["GITHUB_TOKEN"]}`; const slackBotGetToken = () => { const token = process.env["SLACKBOT_TOKEN"]; @@ -78,7 +75,7 @@ const doProcessEndpoints = async work => { const commitReports = await SyncEndpoint( endpoint.GitHubTarget, endpoint, - gitHubCredentials + gitHubToken ); if (commitReports?.length) { //Commits happened diff --git a/wordpress-to-github/common/index.js b/wordpress-to-github/common/index.js index 6f60c83..ff31027 100644 --- a/wordpress-to-github/common/index.js +++ b/wordpress-to-github/common/index.js @@ -39,7 +39,6 @@ const fetchRetry = require("fetch-retry")(require("node-fetch/lib"), { /** * @typedef {{Owner:string, Repo:string, Branch:string, ConfigPath:string}} GitHubTarget - * @typedef {{token:string}} GitHubCredentials * @typedef {{url:string,tags_exclude:string[]}} WordpressSource * @typedef {{width:number,path:string}} WordpressMediaSize */ diff --git a/wordpress-to-github/index.js b/wordpress-to-github/index.js index 518001c..9c4e1ba 100644 --- a/wordpress-to-github/index.js +++ b/wordpress-to-github/index.js @@ -16,7 +16,6 @@ const { pathFromMediaSourceUrl, addMediaSection, GitHubTarget, - GitHubCredentials, EndpointConfigData, SourceEndpointConfigData, WordpressApiDateCacheItem, @@ -73,12 +72,12 @@ const addToReport = (Report, Tree) => { * * @param {GitHubTarget} gitHubTarget * @param {SourceEndpointConfigData} sourceEndpointConfig - * @param {GitHubCredentials} gitHubCredentials + * @param {string} gitHubToken */ const SyncEndpoint = async ( gitHubTarget, sourceEndpointConfig, - gitHubCredentials + gitHubToken ) => { /** @type {GithubCompare[]} */ const report = []; @@ -89,10 +88,7 @@ const SyncEndpoint = async ( repo: gitHubTarget.Repo }; - const configTree = new GitHubTreePush( - gitHubCredentials.token, - configTreeConfig - ); + const configTree = new GitHubTreePush(gitHubToken, configTreeConfig); //https://docs.github.com/en/rest/reference/repos#contents const endpointConfigResponse = await configTree.__fetchResponse( @@ -212,7 +208,7 @@ const SyncEndpoint = async ( .join("/"); const fileName = endpointConfig.GeneralFilePath.split("/").slice(-1)[0]; - const generalTree = new GitHubTreePush(gitHubCredentials.token, { + const generalTree = new GitHubTreePush(gitHubToken, { ...configTreeConfig, path: filePath, commit_message: commitTitleGeneral @@ -229,7 +225,7 @@ const SyncEndpoint = async ( // MEDIA if (endpointConfig.MediaPath && mediaMap && allMedia) { - const mediaTree = new GitHubTreePush(gitHubCredentials.token, { + const mediaTree = new GitHubTreePush(gitHubToken, { ...configTreeConfig, path: endpointConfig.MediaPath, commit_message: commitTitleMedia, @@ -381,7 +377,7 @@ const SyncEndpoint = async ( fieldMetaRefUrl ) => { if (path && WordPressRows?.length) { - const objectTree = new GitHubTreePush(gitHubCredentials.token, { + const objectTree = new GitHubTreePush(gitHubToken, { ...configTreeConfig, path, commit_message, @@ -465,7 +461,7 @@ const SyncEndpoint = async ( // Create and commit a git tree for each set of files. for (let [folderName, fileMap] of Object.entries(apiRequestsByFolder)) { - const requestsTree = new GitHubTreePush(gitHubCredentials.token, { + const requestsTree = new GitHubTreePush(gitHubToken, { ...configTreeConfig, path: folderName, commit_message: commitTitleApiRequests, From dd231ad2c16a96bef9224002c3f8c0f796db97cf Mon Sep 17 00:00:00 2001 From: Carter Medlin Date: Thu, 9 Dec 2021 17:32:46 -0800 Subject: [PATCH 18/18] token simplify --- WordpressSync/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WordpressSync/index.js b/WordpressSync/index.js index ad74e14..55b2a9a 100644 --- a/WordpressSync/index.js +++ b/WordpressSync/index.js @@ -10,7 +10,7 @@ const endPointsJson = require("./endpoints.json"); /** @type {SourceEndpointConfigData[]} */ const endpoints = endPointsJson.data.projects; -const gitHubToken = `${process.env["GITHUB_TOKEN"]}`; +const gitHubToken = process.env["GITHUB_TOKEN"]; const slackBotGetToken = () => { const token = process.env["SLACKBOT_TOKEN"];