diff --git a/scripts/catalog.js b/scripts/catalog.js index 4a138b8b73..54711f6c5b 100644 --- a/scripts/catalog.js +++ b/scripts/catalog.js @@ -99,7 +99,6 @@ const getConnectionModes = (destination) => { */ const doesCatalogItemExist = (item) => { const docsPath = `src/${item.url}` - if (!fs.existsSync(docsPath)) { console.log(`${item.slug} does not exist: ${docsPath}`) let content =`---\ntitle: '${item.display_name} Source'\nhidden: true\n---` @@ -117,6 +116,11 @@ const doesCatalogItemExist = (item) => { } const isCatalogItemHidden = (itemURL) => { + if(!fs.existsSync(path.resolve('src', itemURL, 'index.md'))) + { + console.log("No file exists at " + itemURL + ".") + return true + } const f = fm(fs.readFileSync(path.resolve('src', itemURL, 'index.md'), 'utf8')); if (f.attributes.hidden) return true return false @@ -129,6 +133,7 @@ const updateSources = async () => { let sourceCategories = [] let nextPageToken = null + console.log("Updating Sources catalog from ConfigAPI") while (nextPageToken !== "") { const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/sources`, nextPageToken) sources = sources.concat(res.sources) @@ -177,6 +182,7 @@ const updateSources = async () => { } sourcesUpdated.push(updatedSource) doesCatalogItemExist(updatedSource) + // add unique source categories to set source.categories.reduce((s, e) => s.add(e), categories); }) @@ -190,6 +196,7 @@ const updateSources = async () => { }) // Create source catalog yaml file + console.log("Writing Source catalog file") const options = { noArrayIndent: true }; var todayDate = new Date().toISOString().slice(0,10); output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n" @@ -198,6 +205,7 @@ const updateSources = async () => { fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/sources.yml`), output); // Create source-category mapping yaml file + console.log("Writing Source category list") var todayDate = new Date().toISOString().slice(0,10); output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n" output += "# source cateogries last updated " + todayDate + " \n"; @@ -211,20 +219,20 @@ const updateDestinations = async () => { let destinationCategories = [] let categories = new Set() let nextPageToken = null - while (nextPageToken !== "") { const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/destinations`, nextPageToken) destinations = destinations.concat(res.destinations) nextPageToken = res.next_page_token } + console.log("Updating Destinations catalog from ConfigAPI") destinations.sort((a, b) => { if(a.display_name < b.display_name) { return -1; } if(a.display_name > b.display_name) { return 1; } return 0; }) + console.log("Sorting raw Desintation catalog entries ConfigAPI") destinations.forEach(destination => { let slug = slugify(destination.display_name) - let tempCategories = [destination.categories.primary, destination.categories.secondary, ...destination.categories.additional] tempCategories = tempCategories.filter(category => category != '') @@ -244,6 +252,7 @@ const updateDestinations = async () => { if(a.display_name > b.display_name) { return 1; } return 0; }) + settings.forEach(setting => { if (setting.settings.length > 0) { setting.settings.sort((a, b) => { @@ -296,6 +305,7 @@ const updateDestinations = async () => { }) // Create destination catalog yaml file + console.log("Writing Destination catalog file") const options = { noArrayIndent: true }; output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n" var todayDate = new Date().toISOString().slice(0,10); @@ -303,13 +313,14 @@ const updateDestinations = async () => { output += yaml.safeDump({ items: destinationsUpdated }, options); fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/destinations.yml`), output); - // Create destination-category mapping yaml file + // Create destination-category mapping yaml file + console.log("Writing Destination category list") output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n" var todayDate = new Date().toISOString().slice(0,10); output += "# destination categories last updated " + todayDate + " \n"; output += yaml.safeDump({ items: destinationCategories }, options); fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/destination_categories.yml`), output); } - +console.log("Hello world") updateSources() updateDestinations()