Skip to content

Commit

Permalink
Merge f9f5ff3 into 51c02df
Browse files Browse the repository at this point in the history
  • Loading branch information
@datakurre
datakurre committed Jul 10, 2018
2 parents 51c02df + f9f5ff3 commit a015dab
Show file tree
Hide file tree
Showing 36 changed files with 216 additions and 116 deletions.
66 changes: 48 additions & 18 deletions src/gatsby-node.js
View file
Expand Up @@ -56,25 +56,38 @@ const fetchAllItems = async (baseUrl, token, searchParams) => {
};

// Process HTML data using react-html-parser
const processHtml = (html, baseUrl) => {
const processHtml = (html, baseUrl, path, backlinks) => {
const transform = (node, index) => {
// Replace hyperlinks with relative links
if (node.type === 'tag' && node.name === 'a') {
if (node.attribs.href && node.attribs.href.startsWith(baseUrl)) {
node.attribs.to = normalizePath(node.attribs.href.split(baseUrl)[1]);
node.attribs.href = null;
node.name = 'Link';
if (!backlinks[node.attribs.to]) {
backlinks[node.attribs.to] = [path];
} else {
backlinks[node.attribs.to].push(path);
}
return convertNodeToElement(node, index, transform);
}
}

// Replace image src with relative paths
// if (node.type === 'tag' && node.name === 'img') {
// if (node.attribs.src && node.attribs.src.startsWith(baseUrl)) {
// node.attribs.src = node.attribs.src.split(baseUrl)[1];
// return convertNodeToElement(node, index, transform);
// }
// }
if (node.type === 'tag' && node.name === 'img') {
if (node.attribs.src && node.attribs.src.startsWith(baseUrl)) {
node.attribs.src = normalizePath(
node.attribs.src.split(baseUrl)[1].split('/@@images')[0]
);
node.name = 'Img';
if (!backlinks[node.attribs.src]) {
backlinks[node.attribs.src] = [path];
} else {
backlinks[node.attribs.src].push(path);
}
return convertNodeToElement(node, index, transform);
}
}
};

const options = {
Expand All @@ -86,7 +99,7 @@ const processHtml = (html, baseUrl) => {
};

// Process data to pass it on to nodes
const processData = (data, baseUrl) => {
const processData = (data, baseUrl, backlinks) => {
let node = {
internal: {
contentDigest: createContentDigest(data),
Expand Down Expand Up @@ -131,17 +144,30 @@ const processData = (data, baseUrl) => {
}
});

if (node.text) {
if (node.text['content-type'] === 'text/html') {
node.text.react = processHtml(node.text.data, baseUrl);
}
}

// Add node _path variable to be used similar to slug
node._path = normalizePath(
urlWithoutParameters(data['@id']).split(baseUrl)[1]
);

// Add array of backlinks
if (!backlinks[node._path]) {
backlinks[node._path] = node._backlinks = []; // create new container
} else {
node._backlinks = backlinks[node._path]; // merge with found backlinks
}

// Transform HTML string into serialized React tree
if (node.text) {
if (node.text['content-type'] === 'text/html') {
node.text.react = processHtml(
node.text.data,
baseUrl,
node._path,
backlinks
);
}
}

// Tree hierarchy in nodes
node.id = urlWithoutParameters(data['@id']);
node.parent = data.parent['@id'] ? data.parent['@id'] : null;
Expand Down Expand Up @@ -236,14 +262,17 @@ const processNodesUsingSearchTraversal = async (
})
);

// Mutable container for collecting all backlinks
let backlinks = {};

logMessage('Creating node structure', showLogs);
const nodes = items.map(item => {
return processData(item, baseUrl);
return processData(item, baseUrl, backlinks);
});

// Fetch data, process node for PloneSite
const ploneSite = await fetchData(baseUrl, token, expansions);
const ploneSiteNode = processData(ploneSite, baseUrl);
const ploneSiteNode = processData(ploneSite, baseUrl, backlinks);
// Push to nodes array
nodes.push(ploneSiteNode);

Expand All @@ -257,7 +286,8 @@ const processNodesUsingRecursion = async (
expansions,
showLogs
) => {
let nodes = [];
let nodes = [],
backlinks = {};
const queue = [baseUrl];

logMessage('Traversing the site and fetching data', showLogs);
Expand All @@ -270,7 +300,7 @@ const processNodesUsingRecursion = async (
: [];
queue.push(...children);

nodes.push(processData(itemData, baseUrl));
nodes.push(processData(itemData, baseUrl, backlinks));
}

return nodes;
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs.json
View file
Expand Up @@ -15,10 +15,10 @@
},
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
"UID": "c532b7838eec46369509c152ed8481d3",
"UID": "77fdffe3da7e492aa7b420074768b596",
"allow_discussion": false,
"contributors": [],
"created": "2018-07-05T06:02:20+00:00",
"created": "2018-07-10T06:16:33+00:00",
"creators": [
"admin"
],
Expand Down Expand Up @@ -96,7 +96,7 @@
"items_total": 9,
"language": "en",
"layout": "listing_view",
"modified": "2018-07-05T06:02:33+00:00",
"modified": "2018-07-10T06:16:47+00:00",
"nextPreviousEnabled": false,
"parent": {
"@id": "http://localhost:8080/Plone",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/authentication.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/authentication",
"@type": "Document",
"UID": "cd2be940b47040a48c90f3132ddab917",
"UID": "d3bff246ff514032ae5e08aa6992b189",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:25+00:00",
"created": "2018-07-10T06:16:38+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:25+00:00",
"modified": "2018-07-10T06:16:38+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/expansions.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/expansions",
"@type": "Document",
"UID": "31bce54308884b72a021d206506efaf6",
"UID": "79cbfeb773e645f68dbd74a95c12e550",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:28+00:00",
"created": "2018-07-10T06:16:41+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:28+00:00",
"modified": "2018-07-10T06:16:41+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/gatsby_v2.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/gatsby_v2",
"@type": "Document",
"UID": "7a32c0f60da947b9b02369f18d4ae255",
"UID": "ea7066b90f9b49f99a7c18ba0effb627",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:33+00:00",
"created": "2018-07-10T06:16:47+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:33+00:00",
"modified": "2018-07-10T06:16:47+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/index.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/index",
"@type": "Document",
"UID": "5a0938ce84f943da8b0e44a21c51b085",
"UID": "a1bc757c9079479c992abd7f50c27b8e",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:22+00:00",
"created": "2018-07-10T06:16:35+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:22+00:00",
"modified": "2018-07-10T06:16:35+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/plugin_options.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/plugin_options",
"@type": "Document",
"UID": "3475ba44c58c43f88e85a6d860d08755",
"UID": "ad064cdcda77459dbe4a7adb2db4effc",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:23+00:00",
"created": "2018-07-10T06:16:36+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:23+00:00",
"modified": "2018-07-10T06:16:37+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/recursive_traversal.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/recursive_traversal",
"@type": "Document",
"UID": "5f2e09b140fa4df487334c7f659faa15",
"UID": "723649fe837b4c7cb03d322e063ddbd0",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:29+00:00",
"created": "2018-07-10T06:16:43+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:29+00:00",
"modified": "2018-07-10T06:16:43+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/search_parameters.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/search_parameters",
"@type": "Document",
"UID": "7c85a2924d0649008c0f82d122fc0343",
"UID": "4d498abd2e70426ab95a4737f49658fc",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:32+00:00",
"created": "2018-07-10T06:16:46+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:32+00:00",
"modified": "2018-07-10T06:16:46+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/search_traversal.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/search_traversal",
"@type": "Document",
"UID": "4defdf92b91c4571b5b59bc77d0c8b7a",
"UID": "e97bec05082f464eb06ef707c34b4445",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:30+00:00",
"created": "2018-07-10T06:16:44+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:31+00:00",
"modified": "2018-07-10T06:16:44+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/docs/tree-hierarchy.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/docs/tree-hierarchy",
"@type": "Document",
"UID": "28702aaf26b14dfbaa634f82b275f5ba",
"UID": "b107fb3551524d3ea4df42de5cbd79f5",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:26+00:00",
"created": "2018-07-10T06:16:40+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:26+00:00",
"modified": "2018-07-10T06:16:40+00:00",
"parent": {
"@id": "http://localhost:8080/Plone/docs",
"@type": "Folder",
Expand Down
6 changes: 3 additions & 3 deletions tests/gatsby-starter-default/fixture/index.json
View file
Expand Up @@ -15,11 +15,11 @@
},
"@id": "http://localhost:8080/Plone/index",
"@type": "Document",
"UID": "b043bb0696de48b99a7d515430ccc4ca",
"UID": "c9a5f557e20f4ce78f54348d255190da",
"allow_discussion": false,
"changeNote": "",
"contributors": [],
"created": "2018-07-05T06:02:35+00:00",
"created": "2018-07-10T06:16:49+00:00",
"creators": [
"admin"
],
Expand All @@ -31,7 +31,7 @@
"is_folderish": false,
"language": "",
"layout": "document_view",
"modified": "2018-07-05T06:02:35+00:00",
"modified": "2018-07-10T06:16:49+00:00",
"parent": {
"@id": "http://localhost:8080/Plone",
"@type": "Plone Site",
Expand Down

0 comments on commit a015dab

Please sign in to comment.