Attempt to migrate external repo docs to new format

.gitignore

@@ -7,4 +7,6 @@
 /htmltest/
 /tmp/
 node_modules/
+/scripts/repos/
+/scripts/temp/
 *.pyc

scripts/changeLink.js

@@ -1,18 +1,27 @@
 module.exports = function changeLink(relative) {
-    // replace segment camelcase
-    // /Wurst/WALHasenPengPuff => /Wurst/WALHasen-peng-puff
-    relative = relative.replace(/(?<![A-Z\/])([A-Z])/g, function(x, char) {
-        return "-" + char.toLowerCase();
+    let origLink = relative;
+
+    // replace segment camel case
+    // /Wurst/WALHasenPengPuff => /Wurst/WALHasen-Peng-Puff
+    relative = relative.replace(/(?<![A-Z\/\\])([A-Z])/g, function(x, char) {
+        return "-" + char;
     });
     if (relative.startsWith("-")) {
         relative = relative.substr(1);
     }
     relative = relative.toLowerCase();
-    let fileName = relative.replace(/\/(readme\.md|index\.html)$/, '').replace(/\.(md|html)/, '').replace(/[^a-z0-9]+/g, '-') + '.md';
-
+
+    let fileName = relative
+        .replace(/[\/\\](?:readme\.md|index\.html)$/, '')
+        .replace(/\.(?:md|html)/, '')
+        .replace(/[^a-z0-9]+/g, '-') + '.md';
+
     // root level index
     if (fileName == 'readme.md') {
         fileName = 'index.md'
     }
+
+    //console.log(`${origLink} => ${fileName}`)
+
     return fileName;
 }

scripts/fetchRefs.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Must not set -e or ssh stuff will abort the script
+#set -e
+
+GITAUTH="$1"
+
+if test "${GITAUTH}" == "migrate"; then
+    # Fast path, migration only
+    SOURCES=()
+else
+    SOURCES=(
+        "https://@github.com/arangodb-helper/arangodb.git;arangodb-starter;master;docs/;Manual/"
+        "https://@github.com/arangodb/arangosync.git;arangosync;master;docs/;Manual/"
+        "https://@github.com/arangodb/kube-arangodb.git;kube-arangodb;master;docs/;Manual/"
+        "https://@github.com/arangodb/arangodb-java-driver.git;arangodb-java-driver;master;docs/;Drivers/"
+        "https://@github.com/arangodb/arangodb-php.git;arangodb-php;devel;docs/;Drivers/"
+        "https://@github.com/arangodb/arangodb-spark-connector.git;arangodb-spark-connector;master;docs/;Drivers/"
+        "https://@github.com/arangodb/arangojs.git;arangojs;master;docs/;Drivers/"
+        "https://@github.com/arangodb/go-driver.git;go-driver;master;docs/;Drivers/"
+        "https://@github.com/arangodb/spring-data.git;spring-data;master;docs/;Drivers/"
+    )
+fi
+
+# Ensure agent is running
+if test "${GITAUTH}" == "git"; then
+    ssh-add -l &> /dev/null
+    if [ "$?" == 2 ]; then
+        # Could not open a connection to your authentication agent.
+
+        # Load stored agent connection info
+        test -r ~/.ssh-agent && eval "$(<~/.ssh-agent)" > /dev/null
+        ssh-add -l &> /dev/null
+        if [ "$?" == 2 ]; then
+            # Start agent and store connection info
+            (umask 066; ssh-agent > ~/.ssh-agent)
+            eval "$(<~/.ssh-agent)" > /dev/null
+        fi
+    fi
+
+    # Load identities
+    ssh-add -l &> /dev/null
+    if [ "$?" == 1 ]; then
+        # The agent has no identities.
+        ssh-add
+    fi
+fi
+
+for source in "${SOURCES[@]}"; do
+
+    REPO=$(echo     "$source" |cut -d ';' -f 1)
+    CLONEDIR=$(echo "$source" |cut -d ';' -f 2)
+    BRANCH=$(echo   "$source" |cut -d ';' -f 3)
+    SUBDIR=$(echo   "$source" |cut -d ';' -f 4)
+    DST=$(echo      "$source" |cut -d ';' -f 5)
+
+    CODIR="repos/${CLONEDIR}"
+    AUTHREPO="${REPO/@/${GITAUTH}@}"
+    if test -d "${CODIR}"; then
+        (
+            cd "${CODIR}"
+            git pull --all
+        )
+    else
+        if test "${GITAUTH}" == "git"; then
+            AUTHREPO=$(echo "${AUTHREPO}" | sed -e "s;github.com/;github.com:;" -e "s;https://;;" )
+        fi
+        git clone "${AUTHREPO}" "${CODIR}"
+    fi
+
+    if [ -z "${BRANCH}" ]; then
+        echo "No branch specified for ${CLONEDIR}"
+        exit 1
+    fi
+
+    # checkout branch and pull=merge origin
+    (cd "${CODIR}" && git checkout "${BRANCH}" && git pull)
+
+    for oneMD in $(cd "${CODIR}/${SUBDIR}/${DST}"; find "./" -type f |sed "s;\./;;"); do
+        NAME=$(basename "${oneMD}")
+        MDSUBDIR="${oneMD/${NAME}/}"
+        DSTDIR="temp/${DST}/${MDSUBDIR}"
+        if test ! -d "${DSTDIR}"; then
+            mkdir -p "${DSTDIR}"
+        fi
+        sourcefile="${CODIR}/${SUBDIR}/${DST}/${oneMD}"
+        targetfile="${DSTDIR}/${NAME}"
+            cp "$sourcefile" "$targetfile"
+    done
+done
+
+folder="3.5"
+echo "Migrating to ${folder}"
+node migrate.js temp/Drivers ../${folder}/drivers > /dev/null
+node migrate.js temp/Manual ../${folder} > /dev/null

scripts/migrate.js

@@ -10,7 +10,9 @@ const fetch = require('node-fetch');
 
 async function migrateMds(basePath, targetPath) {
     const absoluteBasePath = await fs.realpath(basePath);
-    const paths = (await globby([path.join(basePath, "**/*.md")])).filter(filePath => {
+    // NOTE globby >= 10 does not support backslashes anymore
+    const searchPath = path.join(basePath, "**/*.md").replace(/\\/g, '/')
+    const paths = (await globby([searchPath])).filter(filePath => {
         return path.relative(basePath, filePath).toLowerCase() !== 'summary.md';
     });
 
@@ -23,7 +25,7 @@ async function migrateMds(basePath, targetPath) {
         let book = path.basename(basePath);
         const version = targetPath.split("/")[0];
         let urlVersion = version;
-        if (urlVersion == "3.5") {
+        if (urlVersion == "3.6") {
             urlVersion = "devel";
         }
 
@@ -63,7 +65,7 @@ async function migrateMds(basePath, targetPath) {
                 }
             } else {
                 description = paragraph.textContent
-                    .replace(/\n/g, '')
+                    .replace(/\n/g, ' ')
                     .replace(/(.*?)[\.:].*/ms, '\$1')
                     .replace(/{%\s*hint[^%]*\s*%}/, '')
                     .replace(/{%\s*endhint\s*%}/, '')
@@ -87,6 +89,17 @@ async function migrateMds(basePath, targetPath) {
         }
         content = content.replace("ArangoDB VERSION_NUMBER", "ArangoDB {{ site.data.versions[page.version.name] }}");
 
+        // HACK: Fix cross-links
+        content = content.replace(/\]\(https:\/\/docs\.arangodb\.com\/latest\/(.*?)\)/g, (fullMatch, link) => {
+            link = path.relative(relative, link).replace(/\\/g, '/').replace(/\/index\.html/, '.html').replace(/\/README.md/, '.html');
+            return `](${link})`;
+        });
+        content = content.replace(/\]\(https:\/\/www\.arangodb\.com\/docs\/stable\/(.*?)\)/g, (fullMatch, link) => {
+            link = path.relative(relative, link).replace(/\\/g, '/').replace(/\/index\.html/, '.html').replace(/\/README.md/, '.html');
+            console.dir({fullMatch, link});
+            return `](${link})`;
+        });
+
         // replace all md links with their changed link
         content = content.replace(/\]\((?!https?:)(.*?\.(html|md))(#[^\)]+)?\)/g, (x, link, fileExt, anchor) => {
             if (!link.startsWith('/')) {
@@ -156,7 +169,9 @@ async function migrateMds(basePath, targetPath) {
 
 // verified beforehand that all imagenames are unique over all directories
 async function migrateImages(basePath, targetPath) {
-    const paths = await globby([path.join(basePath + "/**/*.png")]);
+    // NOTE globby >= 10 does not support backslashes anymore
+    searchPath = path.join(basePath + "/**/*.png").replace(/\\/g, '/')
+    const paths = await globby([searchPath]);
     let imagePath;
     if (path.basename(basePath) == "Manual" || path.basename(basePath) == "Users") {
         imagePath = path.join(targetPath, "images");

scripts/package.json

@@ -9,11 +9,11 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "globby": "^9.1.0",
-    "jsdom": "^14.0.0",
-    "marked-it-core": "^0.13.1",
+    "globby": "^10.0.1",
+    "jsdom": "^15.2.1",
+    "marked-it-core": "^0.14.0",
     "mkdirp-promise": "^5.0.1",
-    "node-fetch": "^2.3.0",
-    "showdown": "^1.9.0"
+    "node-fetch": "^2.6.0",
+    "showdown": "^1.9.1"
   }
 }