Support external git repositories as build dependencies

.travis.yml

@@ -6,3 +6,9 @@ node_js:
   - "8.6"
 after_success:
   - npm run coveralls
+addons: # for nodegit deps
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - libstdc++-4.9-dev

notes.md

@@ -75,6 +75,8 @@
 * version selection
 * custom styling for a changelog.md
 * overview page like https://timber.io/docs
+* feedback area https://cl.ly/1h0X1c3b1Q1Z, might require a hosted plan
+* potentially have a special API section that parses methods like https://doc.esdoc.org/github.com/jy95/torrent-files-library/typedef/index.html#static-typedef-customParsingFunction
 
 ### Components
 

package.json

@@ -66,6 +66,7 @@
 		"html-minifier": "^3.5.15",
 		"markdown-to-jsx": "^6.6.2",
 		"minimist": "^1.2.0",
+		"nodegit": "^0.22.0",
 		"progress": "^2.0.0",
 		"prop-types": "^15.6.1",
 		"react": "^16.3.2",

src/cmds/serve.js

@@ -25,4 +25,5 @@ module.exports.menu = `
 
   ${styles.title('Options')}
 
-    ${styles.subnote('no options yet')}`
+    ${styles.subnote('no options yet')}
+`

src/core/externals.js

@@ -0,0 +1,105 @@
+const fs = require('fs-extra')
+const path = require('path')
+const git = require('nodegit')
+const { log, warn, error } = require('../utils/emit')
+
+// Warnings for missing source information
+const warnings = {
+  url: 'Source must have a "url" defined, skipping...',
+  name: 'Source must have a "name" defined, skipping...',
+  branch: 'No source branch specified, defaulting to master...',
+  root: 'No source root specified, defaulting to "/docs"...'
+}
+
+// Default source attributes
+const defaults = {
+  branch: 'master',
+  root: 'docs/'
+}
+
+// Tmp directories uses to clone and extract docs
+const directories = {
+  tmp: '.gitdocs_tmp',
+  externals: '.gitdocs_externals'
+}
+
+function checkoutBranch (repo, source) {
+  return repo
+    .getBranch(`refs/remotes/origin/${source.branch}`)
+    .then(b => repo.checkoutRef(b))
+}
+
+function cloneExternals (sources) {
+  // Ensure the externals directory is cleared out
+  fs.removeSync(directories.externals)
+
+  // Map sources into an array of clone requests
+  const requests = sources.map(async (s) => {
+    // Check for proper source configuration
+    if (!s.url) warn(warnings.url)
+    if (!s.name) warn(warnings.name)
+
+    // Print loading status
+    log(`Fetching ${s.url}...`)
+
+    // Override defaults with source config
+    const source = { ...defaults, ...s }
+
+    // Clone the source folder to our tmp directory
+    const repo = await git.Clone(source.url, `${directories.externals}/${source.name}`)
+    return checkoutBranch(repo, source)
+  })
+
+  // Wait for all cloning to finish
+  return Promise
+    .all(requests)
+    .catch(err => error(`Clone error ${err}`))
+}
+
+function extractDocs (sources) {
+  // Valid external sources will be collected here
+  const externals = []
+
+  // Ensure the tmp directory is cleared out
+  fs.removeSync(directories.tmp)
+
+  sources.forEach(s => {
+    // Get the root path for the external source
+    const rootPath = s.root || defaults.root
+
+    // Get the actual location of the docs
+    const docsRoot = path.resolve(
+      `${directories.externals}/${s.name}`,
+      rootPath
+    )
+
+    // Warn if the docs root doesn't exist
+    if (!fs.existsSync(docsRoot)) {
+      return warn(`No docs root found for ${s.name || 'source'}, skipping...`)
+    }
+
+    // Add to the list of valid external doc sources
+    externals.push(s.name)
+
+    // Move the external docs repo to our tmp folder
+    fs.copySync(docsRoot, `${directories.tmp}/${s.name}`)
+  })
+
+  return externals
+}
+
+module.exports = async (config) => {
+  // If we have no external repositories defined return early
+  if (!config.sources) return false
+
+  // Clone all external sources into externals folder
+  await cloneExternals(config.sources)
+
+  // Extract docs directories and move to gitdocs tmp folder
+  const externals = extractDocs(config.sources)
+
+  // Ensure our externals directory is cleared out
+  fs.removeSync(directories.externals)
+
+  return externals
+}

src/core/index.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra')
 const loadSyntax = require('./syntax')
+const getExternals = require('./externals')
 const getManifest = require('./manifest')
 const getCompiler = require('./compiler')
 
@@ -8,7 +9,8 @@ module.exports = async (env, config, bar) => {
   await fs.emptyDir(config.output)
   await loadSyntax(config)
 
-  const manifest = await getManifest(env, config)
+  const externals = await getExternals(config)
+  const manifest = await getManifest(env, config, externals)
 
   // this gets passed to the theme app
   const props = {

src/core/manifest.js

@@ -133,7 +133,7 @@ async function buildManifest (env, opts = {}) {
   }
 }
 
-module.exports = async (env, config) => {
+module.exports = async (env, config, externals) => {
   if (!await fs.pathExists(config.root)) {
     throw new Error(`Could not find root documentation folder: ${config.root}`)
   }