Pull in Sources Catalog from Platform API

Rakefile

@@ -169,6 +169,30 @@ namespace :catalog do
 
   desc 'Updates the source catalog data file based on the current data available in the Platform API'
   task :update_sources do
+    sources = []
+    begin
+      resp = {}
+      next_page_token = nil
+      loop do
+        resp = get_catalog(url: "#{PLATFORM_API_URL}/v1beta/catalog/sources", page_token: next_page_token)
+        next_page_token = resp.delete("next_page_token")
+        sources = (sources << resp['sources']).flatten
+
+        break if next_page_token == ""
+      end
+
+    rescue Exception => e
+      abort e
+    end
+
+    sources.sort_by! { |d| d['name'] }
+
+    File.open("./_data/catalog/sources.yml","w") do |file|
+      file.write "# AUTOGENERATED FROM PLATFORM API. DO NOT EDIT\n"
+      file.write(({"sources" => sources}).to_yaml({ indention: 4, separator: '' }))
+    end
+
+    p "Finished Sources."
   end
 end
 

_data/catalog/cloud_sources.yml

@@ -0,0 +1,133 @@
+---
+sources:
+- name: catalog/sources/adwords
+  display_name: Google Ads
+  description: The Google Ads source pulls impressions, spend, campaign, and click
+    data from your Google Ads account into your data warehouse.
+  categories:
+  - Advertising
+  logos:
+    logo: https://cdn.filepicker.io/api/file/KEnErmDUQKiOWgkvzofp
+  type: third-party
+- name: catalog/sources/desk-com
+  display_name: Desk.com
+  description: Desk.com source will load your Desk.com data into a Postgres, Redshift,
+    or BigQuery warehouse.
+  categories:
+  - Helpdesk
+  logos:
+    logo: https://cdn.filepicker.io/api/file/PsYR3untRVGZVoHUYSpU
+    mark: https://cdn.filepicker.io/api/file/kXfnDxpSSrSp36khgjkM
+  type: third-party
+- name: catalog/sources/facebook-ads
+  display_name: Facebook Ads
+  description: The Facebook Ads source pulls impressions, spend, campaign, and click
+    data from your Facebook account into your data warehouse.
+  categories:
+  - Advertising
+  logos:
+    logo: https://cdn.filepicker.io/api/file/N0wYsVjjQIWoNFPrLyUJ
+    mark: https://cdn.filepicker.io/api/file/Pf1GvLowQD6A5fqJDT51
+  type: third-party
+- name: catalog/sources/hubspot
+  display_name: HubSpot
+  description: When you authenticate with your Hubspot account, we'll pull their core
+    resources to any warehouse you have enabled in your Segment account.
+  categories:
+  - CRM
+  logos:
+    logo: https://cdn.filepicker.io/api/file/618XEphQoWXwBKQGOWNw
+    mark: https://cdn.filepicker.io/api/file/KBNTuJHwTnOUIjvG70ad
+  type: third-party
+- name: catalog/sources/intercom
+  display_name: Intercom
+  description: "Once you have successfully added your credentials we start syncing
+    your Intercom objects to your warehouse. \n\nWe will update Intercom data every
+    ~3 hours."
+  categories:
+  - Email Marketing
+  logos:
+    logo: https://cdn.filepicker.io/api/file/Qokc1wcbQBe1Z0CNcQ8F
+    mark: https://cdn.filepicker.io/api/file/KaaNEw3oQRuOV6VVXnn2
+  type: third-party
+- name: catalog/sources/mailchimp
+  display_name: Mailchimp
+  description: "Once you have successfully added your credentials we start syncing
+    your Mailchimp objects to your warehouse. \n\nWe will update Mailchimp data every
+    ~3 hours."
+  categories:
+  - Email Marketing
+  logos:
+    logo: https://cdn.filepicker.io/api/file/kDUn2hJRWm7lESgxvkg3
+    mark: https://cdn.filepicker.io/api/file/xTMOZrHVRB6MYYjyNIcj
+  type: third-party
+- name: catalog/sources/mandrill
+  display_name: Mandrill
+  categories:
+  - Email Marketing
+  logos:
+    logo: https://cdn.filepicker.io/api/file/xFrM7wotRAWijOcUoQcn
+    mark: https://cdn.filepicker.io/api/file/SBd4NRhHRVuXBZJy1f1f
+  type: third-party
+- name: catalog/sources/salesforce
+  display_name: Salesforce
+  description: "When you enter your Salesforce credentials and hit save, we will sync
+    Salesforce standard objects (and their corresponding properties) to any databases
+    you have turned on. \n\nSalesforce data will reflect the state of the object at
+    the time of the sync.\n\nWe will run an update every ~3 hours."
+  categories:
+  - CRM
+  logos:
+    logo: https://d3hotuclm6if1r.cloudfront.net/logos/salesforce-default.svg
+  type: third-party
+- name: catalog/sources/sendgrid
+  display_name: SendGrid
+  description: |-
+    Once you have successfully added your credentials we start syncing your SendGrid objects to your warehouse.
+
+    We will update SendGrid data every ~3 hours.
+  categories:
+  - Email Marketing
+  logos:
+    logo: https://d3hotuclm6if1r.cloudfront.net/logos/sendgrid-default.svg
+    mark: https://cdn.filepicker.io/api/file/v23t5ZESCSMeDB0JMvH7
+  type: third-party
+- name: catalog/sources/stripe
+  display_name: Stripe
+  description: Once you have successfully OAuth’d into Stripe, we will begin syncing
+    Stripe objects (and their corresponding properties) to any databases you have
+    turned on (to turn on a database, navigate to the database tab in the navigation
+    pane on the left).
+  categories:
+  - Payments
+  logos:
+    logo: https://cdn.filepicker.io/api/file/7BXASJF8ReVG9pfQCX9Z
+    mark: https://cdn.filepicker.io/api/file/oVSkzKHQ96hIQkbK18ib
+  type: third-party
+- name: catalog/sources/twilio
+  display_name: Twilio
+  description: "Once you have successfully added your credentials we start syncing
+    your Twilio objects to your warehouse. \n\nWe will update Twilio data every ~3
+    hours."
+  categories:
+  - SMS & Push Notifications
+  logos:
+    logo: https://cdn.filepicker.io/api/file/pn1r8UNcTiyFw4m4lqy3
+    mark: https://cdn.filepicker.io/api/file/FBoLevGrQECujcvAjU9g
+  type: third-party
+- name: catalog/sources/zendesk
+  display_name: Zendesk
+  description: "When you enter your Zendesk credentials and hit save, we will sync
+    standard Zendesk objects (and their corresponding properties) to any databases
+    you have turned on (to turn on a database, navigate to the database tab in the
+    navigation pane on the left).\n\nThese objects will be written into a separate
+    schema, corresponding to your project’s name.\n\nZendesk data will reflect the
+    state of the object at the time of the sync: so if a ticket status goes from `unassigned`
+    to `assigned` between syncs, on its next sync that tickets status will be `assigned`.
+    \n\nWe will update Zendesk data every ~3 hours."
+  categories:
+  - Helpdesk
+  logos:
+    logo: https://d3hotuclm6if1r.cloudfront.net/logos/zendesk-default.svg
+    mark: https://cdn.filepicker.io/api/file/32YThZzWRDmZhbO9AqjE
+  type: third-party

_data/catalog/warehouses.yml

@@ -0,0 +1,51 @@
+---
+warehouses:
+- display_name: Redshift
+  name: catalog/warehouses/redshift
+  slug: redshift
+  public: true
+  logos:
+    logo: https://d3hotuclm6if1r.cloudfront.net/logos/redshift-default.svg
+    mark: ''
+- display_name: BigQuery
+  name: catalog/warehouses/bigquery
+  slug: bigquery
+  public: true
+  logos:
+    logo: https://cdn.filepicker.io/api/file/bDzeRa75SZc6FfgfoUK6
+    mark: https://cdn.filepicker.io/api/file/Vk6iFlMvQeynbg30ZEtt
+- display_name: Postgres
+  name: catalog/warehouses/postgres
+  slug: postgres
+  public: true
+  logos:
+    logo: https://d3hotuclm6if1r.cloudfront.net/logos/postgres-default.svg
+    mark: ''
+- display_name: Snowflake
+  name: catalog/warehouses/snowflake
+  slug: snowflake
+  public: true
+  logos:
+    logo: https://cdn.filepicker.io/api/file/JrQWOYvMRRCVvSHp4HL0
+    mark: https://cdn.filepicker.io/api/file/OBhrGoCRKaSyvAhDX3fw
+- display_name: IBM Db2 Warehouse
+  name: catalog/warehouses/db2
+  slug: db2
+  public: true
+  logos:
+    logo: https://image.ibb.co/i4rS3K/IBM_Db2_logotype_pos_RGB.jpg
+    mark: ''
+- display_name: Azure SQL Database
+  name: catalog/warehouses/azuresqldb
+  slug: azuresqldb
+  public: false
+  logos:
+    logo: https://msftplayground.com/wp-content/uploads/2017/04/logoAzureSql.png
+    mark: ''
+- display_name: Azure SQL Data Warehouse
+  name: catalog/warehouses/azuresqldw
+  slug: azuresqldw
+  public: true
+  logos:
+    logo: https://cdn.filepicker.io/api/file/VKbuWjNjQPKOnOWijFe4
+    mark: https://cdn.filepicker.io/api/file/EUJvt69Q7qMqCvGrVtiu

_data/sidenav/default.yml

@@ -6,6 +6,9 @@ sections:
     title: Spec
   - section_title: Sources
     slug: connections/Sources
+    section:
+    - path: "/connections/sources/catalog"
+      title: Sources catalog
   - section_title: Destinations
     slug: connections/destinations
     section:
@@ -20,6 +23,8 @@ sections:
     section:
       - path: "/connections/warehouses"
         title: Overview
+      - path: "/connections/warehouses/catalog"
+        title: Warehouses catalog
 - section_title: Guides
   section:
   - path: "/guides"

_includes/content/connection-modes.md

@@ -1,7 +1,44 @@
-The first step is to make sure {{ page.title }} supports the source type and connection mode you've chosen to implement. You can learn more about what dictates [the connection modes we support here](https://segment.com/docs/destinations/#connection-modes).
+The first step is to make sure {{ destination.title }} supports the source type and connection mode you've chosen to implement. You can learn more about what dictates [the connection modes we support here](https://segment.com/docs/destinations/#connection-modes).
 
 
-<!-- TODO: populate from catalog API -->
+<!--
+components -> how do we send data
+platforms -> what data do we recognize-->
+
+{% for components in destination %}
+  {% if destination.components.type == "ios" or destination.components.type == "android" %}
+    {% assign has-mobile = true %}
+    {% assign device-mobile = true %}
+  {% endif %}
+  {% if destination.components.type == "browser" %}
+    {% assign has-browser = true %}
+    {% assign device-web = true %}
+  {% endif %}
+  {% if destination.components.type == "server" %}
+    {% assign has-server = true %}
+    {% assign device-server = true %}
+  {% endif %}
+{% endfor %}
+
+<!-- `cloud-web` is complicated -->
+{% if has-browser == true and browserUnbundlingSupported == true && browserUnbundlingPublic == true %}
+  {% assign cloud-web = true %}
+{% elsif has-server == true and has-browser == false and destination.platforms.browser == true %}
+  {% assign cloud-web = true %}
+{% else %}
+  {% assign cloud-web = false %}
+{% endif %}
+
+<!-- cloud-mobile only checks for server? -->
+{% if has-server == true %}
+{% assign cloud-mobile = true %}
+{% endif %}
+
+<!-- cloud-server is also complicated -->
+{% if has-server == true and destaintion.platforms.server == true %}
+{% assign cloud-mobile = true %}
+{% endif %}
+
 <table>
   <tr>
     <th></th>
@@ -11,14 +48,14 @@ The first step is to make sure {{ page.title }} supports the source type and con
   </tr>
   <tr>
     <td>📱 Device-based</td>
-    <td>{{#device-web integration}} ✅ {{/device-web}}</td>
-    <td>{{#device-mobile integration}} ✅ {{/device-mobile}}</td>
-    <td>{{#device-server integration}} {{/device-server}}</td>
+    <td>{% if device-web == true %} ✅ {% endif %}</td>
+    <td>{% if device-mobile == true %} ✅ {% endif %}</td>
+    <td>{% if device-server == true %} ✅ {% endif %}</td>
   </tr>
   <tr>
     <td>☁️  Cloud-based</td>
-    <td>{{#cloud-web integration}} ✅ {{/cloud-web}}</td>
-    <td>{{#cloud-mobile integration}} ✅ {{/cloud-mobile}}</td>
-    <td>{{#cloud-server integration}} ✅ {{/cloud-server}}</td>
+    <td>{% if cloud-web == true %} ✅ {% endif %}</td>
+    <td>{% if cloud-mobile == true %} ✅ {% endif %}</td>
+    <td>{% if cloud-server == true %} ✅ {% endif %}</td>
   </tr>
-</table>
+</table>

_includes/content/integration-foot.md

@@ -1,67 +1,44 @@
 <hr/>
 
-{% if integration.methods.identify == true and integration.platforms.server == true %}
-
+<!-- if it uses Identify calls and is available on Server, you can use personas?  -->
+{% if destination.methods.identify == true and destination.platforms.server == true %}
 {% include content/personas.md %}
 
 {% endif %}
 
 
-{% if integration.components %}
+{% if destination.components %}
 
 ## Supported Sources and Connection Modes
 
-<table>
-  <tr>
-    <th></th>
-    <th>Web</th>
-    <th>Mobile</th>
-    <th>Server</th>
-  </tr>
-  <tr>
-    <td>📱 Device-based</td>
-    <td>{% if device-web integration == true %} ✅ {% endif %}</td>
-    <td>{% if device-mobile integration == true %} ✅ {% endif %}</td>
-    <td>{% if device-server integration == true %} ✅ {% endif %}</td>
-  </tr>
-  <tr>
-    <td>☁️  Cloud-based</td>
-    <td>{% if cloud-web integration == true %} ✅ {% endif %}</td>
-    <td>{% if cloud-mobile integration == true %} ✅ {% endif %}</td>
-    <td>{% if cloud-server integration == true %} ✅ {% endif %}</td>
-  </tr>
-</table>
-
-To learn more about about Connection Modes and what dictates which we support, [see here](https://segment.com/docs/integrations/#connection-modes).
-{% endif %}
+{% include content/connection-modes.md %}
 
-{% if integration.browserUnbundlingSupported == true and integration.browserUnbundlingPublic == true %}
-We offer an optional **Cloud-based** Connection Mode for **Web** data with {{ integration.name }}. As a reminder, this removes the {{ integration.name }} javascript library from your site, improving performance.
-{% if integration.browserUnbundlingChangelog == true %} However, there are a few disparities between the Cloud-based and Device-based connection modes to keep in mind brefore enabling it in the Segment app. When you enable the **Cloud-based** destination mode, here's what happens:
+To learn more about about Connection Modes and what dictates which we support, [see here](https://segment.com/docs/destinations/#connection-modes).
 
-<!--I don't know if this even works anymore-->
+{% endif %}
 
-{{ integration.browserUnbundlingIntegrationChangelog }}
-{{ integration.browserUnbundlingChangelog }}
+{% if destination.browserUnbundlingSupported == true and destination.browserUnbundlingPublic == true %}
+We offer an optional **Cloud-based** Connection Mode for **Web** data with {{ destination.name }}. As a reminder, this removes the {{ destination.name }} javascript library from your site, improving performance.
+{% if destination.browserUnbundlingChangelog %} However, there are a few disparities between the Cloud-based and Device-based connection modes to keep in mind before enabling it in the Segment app. When you enable the **Cloud-based** destination mode, here's what happens:
 
+<!-- holy crap lets NOT expose the changelog wtf has anyone looked at that content-->
 {% endif %}
 {% endif %}
 
-{% if has-mobile integration == true and has-server integration == true %}}
-Segment offers an *optional* **Device-based** Connection Mode for **Mobile** data with {{ @root.integration.name }}. If you'd like to use those features that require client-based functionality, follow the steps above to ensure you have packaged the {{ @root.integration.name }} SDK with Segment's.
+{% if destination.platforms.mobile == true and destination.platforms.server == true %}}
+Segment offers an *optional* **Device-based** Connection Mode for **Mobile** data with {{ destination.name }}. If you'd like to use those features that require client-based functionality, follow the steps above to ensure you have packaged the {{ destination.name }} SDK with Segment's.
 {% else %}
-This destination *requires* a **Device-based** Connection Mode for **Mobile** data. Follow the steps above to ensure you have packaged the {{ @root.integration.name }} SDK with Segment's.
+This destination *requires* a **Device-based** Connection Mode for **Mobile** data. Follow the steps above to ensure you have packaged the {{ destination.name }} SDK with Segment's.
 {% endif %}
 
 ## Settings
 
 Segment lets you change these destination settings via your Segment dashboard without having to touch any code.
 
 <!-- I'm not sure how to handle the `each` here  -->
-{% for options in integrations %}
-  {% if hidden == false and label != "Unused"}}
-### {{ option.label}}
-{{{option.description}}}
-  {% endif %}
-{% endif %} <!-- jekyll is complaining when i use an `endfor` here -->
+{% for options in destination %}
+{% if destination.option.hidden == false and destination.option.label != "Unused" %}
+### {{ destination.option.label }}
+{{ destination.option.description }}
+{% endif %}
 {% endfor %}