Skip to content
This repository has been archived by the owner on Sep 7, 2020. It is now read-only.

Commit

Permalink
Move & improve code related to file collector to this plugin (out of …
Browse files Browse the repository at this point in the history 
…the core)
  • Loading branch information
@MoOx
MoOx committed Jun 14, 2018
1 parent 39da989 commit 8a87fd8
Show file tree
Hide file tree
Showing 26 changed files with 513 additions and 630 deletions.
2 changes: 1 addition & 1 deletion flow-typed/handmade/sane.js
View file
Expand Up @@ -3,7 +3,7 @@ declare module "sane" {
filepath: string,
root: string,
stat: Object
) => void;
) => void | Promise<void>;
declare type SaneOptionsType = {
watchman: boolean,
glob: $ReadOnlyArray<string>
Expand Down
8 changes: 3 additions & 5 deletions flow-typed/phenomic.js
View file
Expand Up @@ -181,12 +181,10 @@ declare type PhenomicPlugin = {|
db: PhenomicDB
|}) => mixed,
// collector
collectFile?: ({|
collect?: ({|
db: PhenomicDB,
fileName: string,
parsed: Object,
transformer: PhenomicPlugin
|}) => $ReadOnlyArray<mixed> | Promise<$ReadOnlyArray<mixed>>,
transformers: PhenomicPlugins
|}) => void | Promise<void>,
// bunder
buildForPrerendering?: () => Promise<PhenomicAppType>,
build?: () => PhenomicAssets,
Expand Down
16 changes: 16 additions & 0 deletions packages/core/docs/writing-plugins.md
View file
Expand Up @@ -60,6 +60,22 @@ Plugins interesting part are usually located at
Here are the list of methods that allows you to interact with Phenomic
lifecycle.

### `collect`

The method `collect` is useful if you plan to use [Content API](./api.md). It
allows you to collect your data before rendering and to inject it into the
database.

```js
collect: ({ db, transformers }) => Promise;
```

It receive the database and all transformers plugin as arguments

Used in

* https://github.com/phenomic/phenomic/tree/master/packages/plugin-collector-files/src/index.js

### `supportedFileTypes` + `transform`

The method `transform` is useful if you plan to use [Content API](./api.md). It
Expand Down
5 changes: 0 additions & 5 deletions packages/core/package.json
View file
Expand Up @@ -29,17 +29,12 @@
"dotenv-expand": "^4.2.0",
"express": "^4.14.0",
"get-port": "^2.1.0",
"globby": "^6.1.0",
"log-symbols": "^1.0.2",
"mkdirp": "^0.5.1",
"p-map": "^1.2.0",
"path-to-regexp": "^1.7.0",
"react-dev-utils": "^4.2.1",
"rimraf": "^2.5.4",
"sane": "^1.7.0",
"serve": "^6.5.6",
"socket.io": "^1.7.2",
"socket.io-client": "^1.7.2",
"url": "^0.11.0"
},
"engines": {
Expand Down
86 changes: 7 additions & 79 deletions packages/core/src/commands/build.js
View file
Expand Up @@ -7,91 +7,13 @@ import getPort from "get-port";
import rimraf from "rimraf";
import pMap from "p-map";

import { oneShot } from "../watch";
import processFile from "../injection/processFile";
import createAPIServer from "../api";
import writeFile from "../utils/writeFile";
import createDB from "../db";
import log from "../utils/log";
import getPath from "../utils/getPath";

const debug = require("debug")("phenomic:core:commands:build");

let lastStamp = Date.now();
async function getContent(db, config: PhenomicConfig) {
debug("getting content");
const transformers = config.plugins.filter(
item => typeof item.transform === "function"
);
if (!transformers.length) {
throw Error("Phenomic expects at least a transform plugin");
}
const collectors = config.plugins.filter(
item => typeof item.collectFile === "function"
);
if (!collectors.length) {
throw Error("Phenomic expects at least a collector plugin");
}

await Promise.all(
Object.keys(config.content).map(async contentKey => {
let contentPath;
let globs;
try {
let folder;

// "key(and folder)": ["glob/*"]
if (Array.isArray(config.content[contentKey])) {
folder = path.join(config.path, contentKey);
// $FlowFixMe stfu
globs = config.content[contentKey];
} else if (
config.content[contentKey].root &&
config.content[contentKey].globs
) {
// "key": {root: folder, globs: ["glob/*"] }
folder = path.join(config.path, config.content[contentKey].root);
// $FlowFixMe stfu
globs = config.content[contentKey].globs;
} else {
throw new Error(
"Unexpected config for 'content' option: " +
config.content[contentKey].toString()
);
}

contentPath = await getPath(folder);
} catch (e) {
log.warn(
`no '${
contentKey
}' folder found or unable to read files. Please create and put files in this folder (or double check it) if you want the content to be accessible (eg: markdown or JSON files). `
);
}

if (contentPath) {
const files = oneShot({
path: contentPath,
// $FlowFixMe stfu
patterns: globs
});
await db.destroy();
await Promise.all(
files.map(file =>
processFile({
db,
fileKey: contentKey,
file,
transformers,
collectors
})
)
);
}
})
);
}

async function build(config: PhenomicConfig) {
console.log("⚡️ Hey! Let's get on with it");
debug("cleaning dist");
Expand Down Expand Up @@ -142,7 +64,13 @@ async function build(config: PhenomicConfig) {
"📦 Webpack static build done " + (Date.now() - lastStamp) + "ms"
);
lastStamp = Date.now(); // Retreive content
await getContent(db, config);

const transformers = config.plugins.filter(plugin => plugin.transform);
// collectors
await Promise.all(
config.plugins.map(p => p.collect && p.collect({ db, transformers }))
);

console.log("📝 Content processed " + (Date.now() - lastStamp) + "ms");
lastStamp = Date.now();
const renderers: PhenomicPlugins = config.plugins.filter(p => p.getRoutes);
Expand Down
110 changes: 8 additions & 102 deletions packages/core/src/commands/start.js
View file
@@ -1,18 +1,12 @@
// @flow

import path from "path";

import express from "express";
import socketIO from "socket.io";
import getProcessForPort from "react-dev-utils/getProcessForPort";
import chalk from "chalk";

import createWatcher from "../watch";
import processFile from "../injection/processFile";
import createDB from "../db";
import createAPIServer from "../api";
import log from "../utils/log";
import getPath from "../utils/getPath";

const debug = require("debug")("phenomic:core:commands:start");

Expand Down Expand Up @@ -48,108 +42,20 @@ async function start(config: PhenomicConfig) {
process.env.PHENOMIC_SOCKET_PORT = String(config.socketPort);
debug("starting phenomic server");
const db = createDB(config.db);
const renderers = config.plugins.filter(p => p.getRoutes);
const renderer: PhenomicPlugin = renderers[0];
const transformers = config.plugins.filter(plugin => plugin.transform);
// collectors
await Promise.all(
config.plugins.map(p => p.collect && p.collect({ db, transformers }))
);

const phenomicServer = createAPIServer({
db,
plugins: config.plugins,
rootPath: config.baseUrl.pathname + "phenomic"
});
const bundlerServer = await createDevServer({ config });
const renderers = config.plugins.filter(p => p.getRoutes);
const renderer: PhenomicPlugin = renderers[0];
const transformers = config.plugins.filter(
item => typeof item.transform === "function"
);

if (!transformers.length) {
throw new Error("Phenomic expects at least a transform plugin");
}
const collectors = config.plugins.filter(
item => typeof item.collectFile === "function"
);
if (!collectors.length) {
throw new Error("Phenomic expects at least a collector plugin");
}
const io = socketIO(config.socketPort);

const filesPerContentKey = {};

await Promise.all(
Object.keys(config.content).map(async contentKey => {
try {
let folder;
let globs;

// "key(and folder)": ["glob/*"]
if (Array.isArray(config.content[contentKey])) {
folder = path.join(config.path, contentKey);
// $FlowFixMe stfu
globs = config.content[contentKey];
} else if (
config.content[contentKey].root &&
config.content[contentKey].globs
) {
// "key": {root: folder, globs: ["glob/*"] }
folder = path.join(config.path, config.content[contentKey].root);
// $FlowFixMe stfu
globs = config.content[contentKey].globs;
} else {
throw new Error(
"Unexpected config for 'content' option: " +
config.content[contentKey].toString()
);
}

const watcher = createWatcher({
path: await getPath(folder),
// $FlowFixMe stfu
patterns: globs
});

watcher.onChange(async function(files /* deletedFiles */) {
// currently our db is stupid: we don't do removal
// so instead we nuke the db each times there is a tiny change
// for now it's not creating any problem, but it's clearly something
// we need to improve for HUGE website
// @todo: don't nuke the db and think about a way to remove deleted
// files & related injected data (see collector-files)
filesPerContentKey[contentKey] = files;
debug("watcher onChange event");
try {
await db.destroy();
await Promise.all(
Object.keys(filesPerContentKey).map(
async localContentKey =>
await Promise.all(
filesPerContentKey[localContentKey].map(file => {
return processFile({
db,
fileKey: localContentKey,
file,
transformers,
collectors
});
})
)
)
);
} catch (e) {
setTimeout(() => {
throw e;
}, 1);
}
// note: we could emit faster but does it's worth it?
io.emit("change");
});
} catch (e) {
log.warn(
`no '${
contentKey
}' folder found or unable to read files. Please create and put files in this folder (or double check it) if you want the content to be accessible (eg: markdown or JSON files). `
);
}
})
);

bundlerServer.use(phenomicServer);
// $FlowFixMe flow is lost with async function for express
bundlerServer.get("*", function(req, res) {
Expand Down
84 changes: 0 additions & 84 deletions packages/core/src/injection/processFile.js
View file

This file was deleted.

0 comments on commit 8a87fd8

Please sign in to comment.