Skip to content

Commit

Permalink
Bug/issue 386 graph not matching filesystem order (#449)
Browse files Browse the repository at this point in the history 
* init refactor with synchronous approach working

* robust nested directory output testing

* formatting

* robust nested directory output testing

* graph page object clean up

* graph lifecycle refactor:

* revert transform changes

* fix #455

* fix routing logic

* fix incorrect output when pages have index in the name
  • Loading branch information
@thescientist13
thescientist13 committed Dec 30, 2020
1 parent 5d97a05 commit 1eade4f
Show file tree
Hide file tree
Showing 29 changed files with 374 additions and 249 deletions.
312 changes: 105 additions & 207 deletions packages/cli/src/lifecycles/graph.js
View file
Original file line number Diff line number Diff line change
@@ -1,226 +1,124 @@
#!/usr/bin/env node
const fs = require('fs');
const crypto = require('crypto'); // TODO pretty heavy just for some hashing?
const path = require('path');
const fm = require('front-matter');
const toc = require('markdown-toc');

const createGraphFromPages = async (pagesDir, config) => {
let pages = [];
module.exports = generateGraph = async (compilation) => {

return new Promise(async (resolve, reject) => {
try {
const pagesIndexMap = new Map();
let pagesIndex = 0;

const walkDirectory = async (directory) => {
let files = await fs.promises.readdir(directory);

return Promise.all(files.map((file) => {
const filenameHash = crypto.createHash('md5').update(`${directory}/${file}`).digest('hex');
const filePath = path.join(directory, file);
const stats = fs.statSync(filePath);
const isMdFile = path.extname(file) === '.md';
const ishtmlFile = path.extname(file) === '.html';

// map each page to a (0 based) index based on filesystem order
if (isMdFile || ishtmlFile) {
pagesIndexMap.set(filenameHash, pagesIndex);
pagesIndex += 1;
}

return new Promise(async (resolve, reject) => {
try {

if (isMdFile && !stats.isDirectory()) {
const fileContents = await fs.promises.readFile(filePath, 'utf8');
const { attributes } = fm(fileContents);
let { label, template, title } = attributes;
let { meta } = config;
let mdFile = '';

// if template not set, use default
template = template || 'page';

// get remaining string after user's pages directory
let subDir = filePath.substring(pagesDir.length - 1, filePath.length);

// get index of seperator between remaining subdirectory and the file's name
const seperatorIndex = subDir.lastIndexOf('/');

// get md file's name with extension (for generating to scratch)
let fileName = subDir.substring(seperatorIndex + 1, subDir.length - 3);

// get md file's name without the file extension
let fileRoute = subDir.substring(seperatorIndex, subDir.length - 3);

// determine if this is an index file, if so set route to '/'
let route = fileRoute === '/index' ? '/' : `${fileRoute}/`;

// check if additional nested directories
if (seperatorIndex > 0) {
// get all remaining nested page directories
completeNestedPath = subDir.substring(0, seperatorIndex);

// set route to the nested pages path and file name(without extension)
route = completeNestedPath + route;
mdFile = `.${completeNestedPath}${fileRoute}.md`;
relativeExpectedPath = `'..${completeNestedPath}/${fileName}/${fileName}.js'`;
} else {
mdFile = `.${fileRoute}.md`;
relativeExpectedPath = `'../${fileName}/${fileName}.js'`;
}

// generate a random element tag name
label = label || mdFile.split('/')[mdFile.split('/').length - 1].replace('.md', '');

// set <title></title> element text, override with markdown title
title = title || '';

// create webpack chunk name based on route and page name
const routes = route.lastIndexOf('/') === route.length - 1 && route.lastIndexOf('/') > 0
? route.substring(1, route.length - 1).split('/')
: route.substring(1, route.length).split('/');
let chunkName = 'page';

routes.forEach(subDir => {
chunkName += '--' + subDir;
});

/*
* Variable Definitions
*----------------------
* data: custom frontmatter set per page within frontmatter
* mdFile: path for an md file which will be imported in a generated component
* label: the unique label given to generated component element e.g. <wc-md-somelabel></wc-md-somelabel>
* route: route for a given page's url
* template: page template to use as a base for a generated component (auto appended by -template.js)
* filePath: complete absolute path to a md file
* fileName: file name without extension/path, so that it can be copied to scratch dir with same name
* relativeExpectedPath: relative import path for generated component within a list.js file to later be
* imported into app.js root component
* title: the head <title></title> text
* meta: og graph meta array of objects { property/name, content }
* chunkName: generated chunk name for webpack bundle
*/
const customData = attributes;

// prune "reserved" attributes that are supported by Greenwood
// https://www.greenwoodjs.io/docs/front-matter
delete customData.label;
delete customData.imports;
delete customData.title;
delete customData.template;

/* Menu Query
* Custom front matter - Variable Definitions
* --------------------------------------------------
* menu: the name of the menu in which this item can be listed and queried
* index: the index of this list item within a menu
* linkheadings: flag to tell us where to add page's table of contents as menu items
* tableOfContents: json object containing page's table of contents(list of headings)
*/
// set specific menu to place this page
customData.menu = customData.menu || '';

// set specific index list priority of this item within a menu
customData.index = customData.index || '';

// set flag whether to gather a list of headings on a page as menu items
customData.linkheadings = customData.linkheadings || 0;
customData.tableOfContents = [];

if (customData.linkheadings > 0) {
// parse markdown for table of contents and output to json
customData.tableOfContents = toc(fileContents).json;
customData.tableOfContents.shift();
}
/* ---------End Menu Query-------------------- */

pages[pagesIndexMap.get(filenameHash)] = {
data: customData || {},
mdFile,
label,
route,
template,
filePath,
fileName,
relativeExpectedPath,
title,
meta,
chunkName
};
}

// TODO handle top level root index.html
if (ishtmlFile) {
// const { label, template, title } = attributes;
// const { meta } = config;

pages[pagesIndexMap.get(filenameHash)] = {
data: {},
// label,
// route,
// template,
filePath,
route: file === 'index.html' ? '/' : `${file}/`
// fileName,
// relativeExpectedPath,
// title
// meta,
// chunkName
};
}

if (stats.isDirectory()) {
await walkDirectory(filePath);
resolve();
}
resolve();
} catch (err) {
reject(err);
const { context } = compilation;
const { pagesDir } = context;

const walkDirectoryForPages = function(directory, pages = []) {

fs.readdirSync(directory).forEach((filename) => {
const fullPath = `${directory}/${filename}`.replace('//', '/');

if (fs.statSync(fullPath).isDirectory()) {
pages = walkDirectoryForPages(fullPath, pages);
} else {
const fileContents = fs.readFileSync(fullPath, 'utf8');
const { attributes } = fm(fileContents);
const relativePagePath = fullPath.substring(pagesDir.length - 1, fullPath.length);
const relativeWorkspacePath = directory.replace(process.cwd(), '').replace('/', '');
const template = attributes.template || 'page';
const title = attributes.title || compilation.config.title || '';
const label = attributes.label || filename.split('/')[filename.split('/').length - 1].replace('.md', '').replace('.html', '');
let route = relativePagePath.replace('.md', '').replace('.html', '');

/*
* check if additional nested directories exist to correctly determine route (minus filename)
* examples:
* - pages/index.{html,md} -> /
* - pages/about.{html,md} -> /about/
* - pages/blog/index.{html,md} -> /blog/
* - pages/blog/some-post.{html,md} -> /blog/some-post/
*/
if (relativePagePath.lastIndexOf('/') > 0) {
// https://github.com/ProjectEvergreen/greenwood/issues/455
route = label === 'index' || route.replace('/index', '') === `/${label}`
? route.replace('index', '')
: `${route}/`;
} else {
route = route === '/index'
? '/'
: `${route}/`;
}
});
}));
};

if (fs.existsSync(pagesDir)) {
await walkDirectory(pagesDir);
} else {
pages.push({
route: '/'
// prune "reserved" attributes that are supported by Greenwood
// https://www.greenwoodjs.io/docs/front-matter
const customData = attributes;

delete customData.label;
delete customData.imports;
delete customData.title;
delete customData.template;

/* Menu Query
* Custom front matter - Variable Definitions
* --------------------------------------------------
* menu: the name of the menu in which this item can be listed and queried
* index: the index of this list item within a menu
* linkheadings: flag to tell us where to add page's table of contents as menu items
* tableOfContents: json object containing page's table of contents(list of headings)
*/
// set specific menu to place this page
customData.menu = customData.menu || '';

// set specific index list priority of this item within a menu
customData.index = customData.index || '';

// set flag whether to gather a list of headings on a page as menu items
customData.linkheadings = customData.linkheadings || 0;
customData.tableOfContents = [];

if (customData.linkheadings > 0) {
// parse markdown for table of contents and output to json
customData.tableOfContents = toc(fileContents).json;
customData.tableOfContents.shift();
}
/* ---------End Menu Query-------------------- */

/*
* Graph Properties (per page)
*----------------------
* data: custom page frontmatter
* filename: name of the file
* label: text representation of the filename
* path: path to the file relative to the workspace
* route: URL route for a given page on outputFilePath
* template: page template to use as a base for a generated component
* title: a default value that can be used for <title></title>
*/
pages.push({
data: customData || {},
filename,
label,
path: route === '/' || relativePagePath.lastIndexOf('/') === 0
? `${relativeWorkspacePath}${filename}`
: `${relativeWorkspacePath}/${filename}`,
route,
template,
title
});
}
});
}

resolve(pages);

} catch (err) {
reject(err);
}
});
};

// const generateLabelHash = (label) => {
// const hash = crypto.createHash('sha256');

// hash.update(label);

// let elementLabel = hash.digest('hex');

// elementLabel = elementLabel.substring(elementLabel.length - 15, elementLabel.length);

// return elementLabel;
// };
return pages;
};

module.exports = generateGraph = async (compilation) => {
const graph = fs.existsSync(pagesDir)
? walkDirectoryForPages(pagesDir)
: [{ route: '/' }];

return new Promise(async (resolve, reject) => {
try {
const { context, config } = compilation;
compilation.graph = graph;

compilation.graph = await createGraphFromPages(context.pagesDir, config);
if (!fs.existsSync(context.scratchDir)) {
await fs.promises.mkdir(context.scratchDir);
}

await fs.promises.writeFile(`${path.join(process.cwd(), '.greenwood')}/graph.json`, JSON.stringify(compilation.graph));
await fs.promises.writeFile(`${context.scratchDir}/graph.json`, JSON.stringify(compilation.graph));

resolve(compilation);
} catch (err) {
Expand Down
14 changes: 5 additions & 9 deletions packages/cli/src/lifecycles/serialize.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,12 @@ module.exports = serializeCompilation = async (compilation) => {
try {
return Promise.all(pages.map(async(page) => {
const { route } = page;
const url = route.lastIndexOf('/') === route.length - 1
? route
: `${route}/index.html`;

console.info('serializing page...', url);
console.info('serializing page...', route);

return await browserRunner
.serialize(`${serverUrl}${url}`)
.serialize(`${serverUrl}${route}`)
.then(async (html) => {
let outputPath = `${route.replace('/', '')}/index.html`;
const outputPath = `${outputDir}${route}index.html`;

// TODO allow setup / teardown (e.g. module shims, then remove module-shims)
let htmlModified = html;
Expand All @@ -42,7 +38,7 @@ module.exports = serializeCompilation = async (compilation) => {
});
}

await fsp.writeFile(path.join(outputDir, outputPath), htmlModified);
await fsp.writeFile(outputPath, htmlModified);
});
}));
} catch (e) {
Expand All @@ -60,7 +56,7 @@ module.exports = serializeCompilation = async (compilation) => {
const serverAddress = `http://127.0.0.1:${port}`;

console.info(`Serializing pages at ${serverAddress}`);
console.debug('pages to generate', `\n ${pages.map(page => page.mdFile).join('\n ')}`);
console.debug('pages to generate', `\n ${pages.map(page => page.path).join('\n ')}`);

await runBrowser(serverAddress, pages, outputDir);

Expand Down
Loading

0 comments on commit 1eade4f

Please sign in to comment.