From 7ec087bb9c430752160a2f903e224f8ec0786a57 Mon Sep 17 00:00:00 2001
From: sanscontext <lrubin@segment.com>
Date: Fri, 31 Jan 2020 13:04:07 -0800
Subject: [PATCH 1/2] Added some verbosity, still broken though

---
 scripts/catalog.js | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/scripts/catalog.js b/scripts/catalog.js
index 8838d92d17..17143fe508 100644
--- a/scripts/catalog.js
+++ b/scripts/catalog.js
@@ -99,7 +99,7 @@ const getConnectionModes = (destination) => {
  */
 const doesCatalogItemExist = (item) => {
   const docsPath = `src/${item.url}`
-
+  console.log("Checking if " + item.slug + " exists")
   if (!fs.existsSync(docsPath)) {
     console.log(`${item.slug} does not exist: ${docsPath}`)
     let content =`---\ntitle: '${item.display_name} Source'\nhidden: true\n---`
@@ -117,6 +117,11 @@ const doesCatalogItemExist = (item) => {
 }
 
 const isCatalogItemHidden = (itemURL) => {
+  if(!fs.existsSync(path.resolve('src', itemURL, 'index.md')))
+  {
+  console.log("No file exists at " + itemURL + ".")
+  return true
+  }
   const f = fm(fs.readFileSync(path.resolve('src', itemURL, 'index.md'), 'utf8'));
   if (f.attributes.hidden) return true
   return false
@@ -129,6 +134,7 @@ const updateSources = async () => {
   let sourceCategories = []
   let nextPageToken = null
 
+  console.log("Updating Sources catalog from ConfigAPI")
   while (nextPageToken !== "") {
     const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/sources`, nextPageToken)
     sources = sources.concat(res.sources)
@@ -176,6 +182,7 @@ const updateSources = async () => {
     }
     sourcesUpdated.push(updatedSource)
     doesCatalogItemExist(updatedSource)
+
     // add unique source categories to set
     source.categories.reduce((s, e) => s.add(e), categories);
   })
@@ -189,6 +196,7 @@ const updateSources = async () => {
   })
 
   // Create source catalog yaml file
+  console.log("Writing Source catalog file")
   const options = { noArrayIndent: true };
   var todayDate = new Date().toISOString().slice(0,10);
   output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
@@ -197,6 +205,7 @@ const updateSources = async () => {
   fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/sources.yml`), output);
 
   // Create source-category mapping yaml file
+  console.log("Writing Source category list")
   var todayDate = new Date().toISOString().slice(0,10);
   output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
   output += "# source cateogries last updated " + todayDate + " \n";
@@ -210,20 +219,20 @@ const updateDestinations = async () => {
   let destinationCategories = []
   let categories = new Set()
   let nextPageToken = null
-
   while (nextPageToken !== "") {
     const res = await getCatalog(`${PLATFORM_API_URL}/v1beta/catalog/destinations`, nextPageToken)
     destinations = destinations.concat(res.destinations)
     nextPageToken = res.next_page_token
   }
+  console.log("Updating Destinations catalog from ConfigAPI")
   destinations.sort((a, b) => {
     if(a.display_name < b.display_name) { return -1; }
     if(a.display_name > b.display_name) { return 1; }
     return 0;
   })
+  console.log("Sorting raw Desintation catalog entries ConfigAPI")
   destinations.forEach(destination => {
     let slug = slugify(destination.display_name)
-
     let tempCategories = [destination.categories.primary, destination.categories.secondary, ...destination.categories.additional]
     tempCategories = tempCategories.filter(category => category != '')
 
@@ -243,6 +252,7 @@ const updateDestinations = async () => {
       if(a.display_name > b.display_name) { return 1; }
       return 0;
     })
+
     settings.forEach(setting => {
       if (setting.settings.length > 0) {
         setting.settings.sort((a, b) => {
@@ -295,6 +305,7 @@ const updateDestinations = async () => {
   })
 
   // Create destination catalog yaml file
+  console.log("Writing Destination catalog file")
   const options = { noArrayIndent: true };
   output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
   var todayDate = new Date().toISOString().slice(0,10);
@@ -302,13 +313,14 @@ const updateDestinations = async () => {
   output += yaml.safeDump({ items: destinationsUpdated }, options);
   fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/destinations.yml`), output);
 
-   // Create destination-category mapping yaml file
+  // Create destination-category mapping yaml file
+  console.log("Writing Destination category list")
   output = "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
   var todayDate = new Date().toISOString().slice(0,10);
   output += "# destination categories last updated " + todayDate + " \n";
   output += yaml.safeDump({ items: destinationCategories }, options);
   fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/destination_categories.yml`), output);
 }
-
+console.log("Hello world")
 updateSources()
 updateDestinations()

From 81f593d8569f5c38dbde23b5700dc55b771a4d3c Mon Sep 17 00:00:00 2001
From: sanscontext <lrubin@segment.com>
Date: Fri, 31 Jan 2020 13:10:50 -0800
Subject: [PATCH 2/2] Remove noisy line

---
 scripts/catalog.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/catalog.js b/scripts/catalog.js
index 17143fe508..0502bb9218 100644
--- a/scripts/catalog.js
+++ b/scripts/catalog.js
@@ -99,7 +99,6 @@ const getConnectionModes = (destination) => {
  */
 const doesCatalogItemExist = (item) => {
   const docsPath = `src/${item.url}`
-  console.log("Checking if " + item.slug + " exists")
   if (!fs.existsSync(docsPath)) {
     console.log(`${item.slug} does not exist: ${docsPath}`)
     let content =`---\ntitle: '${item.display_name} Source'\nhidden: true\n---`