Skip to content

Commit

Permalink
New Build System using NodeJS (#3117)
Browse files Browse the repository at this point in the history 
* added build.js

* js babelify and concat, css concat working

* less wip

* live reload css working

* css reload on less change working

* added watch_js

* setup file watcher only in dev mode, don't compile variables.less

* Minify js files using babili

- Add --minify flag

* Set minify to false as default

* [minor] Remove redundant code

* Used subprocess instead of os.system

- Also added experimental flag
  • Loading branch information
@netchampfaris @rmehta
netchampfaris authored and rmehta committed Apr 20, 2017
1 parent 721105b commit 743f7ab
Show file tree
Hide file tree
Showing 10 changed files with 381 additions and 21 deletions.
301 changes: 301 additions & 0 deletions frappe/build.js
View file
@@ -0,0 +1,301 @@
const path = require('path');
const fs = require('fs');
const babel = require('babel-core');
const less = require('less');
const chokidar = require('chokidar');

// for file watcher
const app = require('express')();
const http = require('http').Server(app);
const io = require('socket.io')(http);
const file_watcher_port = 6787;

const p = path.resolve;

// basic setup
const sites_path = p(__dirname, '..', '..', '..', 'sites');
const apps_path = p(__dirname, '..', '..', '..', 'apps'); // the apps folder
const apps_contents = fs.readFileSync(p(sites_path, 'apps.txt'), 'utf8');
const apps = apps_contents.split('\n');
const app_paths = apps.map(app => p(apps_path, app, app)) // base_path of each app
const assets_path = p(sites_path, 'assets');
const build_map = make_build_map();

// command line args
const action = process.argv[2] || '--build';

if (!['--build', '--watch'].includes(action)) {
console.log('Invalid argument: ', action);
return;
}

if (action === '--build') {
const minify = process.argv[3] === '--minify' ? true : false;
build({ minify });
}

if (action === '--watch') {
watch();
}

function build({ minify=false } = {}) {
for (const output_path in build_map) {
pack(output_path, build_map[output_path], minify);
}
}

let socket_connection = false;

function watch() {
http.listen(file_watcher_port, function () {
console.log('file watching on *:', file_watcher_port);
});

compile_less().then(() => {
build();
watch_less(function (filename) {
if(socket_connection) {
io.emit('reload_css', filename);
}
});
watch_js(function (filename) {
if(socket_connection) {
io.emit('reload_js', filename);
}
});
});

io.on('connection', function (socket) {
socket_connection = true;

socket.on('disconnect', function() {
socket_connection = false;
})
});
}

function pack(output_path, inputs, minify) {
const output_type = output_path.split('.').pop();

let output_txt = '';
for (const file of inputs) {

if (!fs.existsSync(file)) {
console.log('File not found: ', file);
continue;
}

let file_content = fs.readFileSync(file, 'utf-8');

if (file.endsWith('.html') && output_type === 'js') {
file_content = html_to_js_template(file, file_content);
}

if(file.endsWith('class.js')) {
file_content = minify_js(file_content, file);
}

if (file.endsWith('.js') && !file.includes('/lib/') && output_type === 'js' && !file.endsWith('class.js')) {
file_content = babelify(file_content, file, minify);
}

if(!minify) {
output_txt += `\n/*\n *\t${file}\n */\n`
}
output_txt += file_content;
}

const target = p(assets_path, output_path);

try {
fs.writeFileSync(target, output_txt);
console.log(`Wrote ${output_path} - ${get_file_size(target)}`);
return target;
} catch (e) {
console.log('Error writing to file', output_path);
console.log(e);
}
}

function babelify(content, path, minify) {
let presets = ['es2015', 'es2016'];
if(minify) {
presets.push('babili'); // new babel minifier
}
try {
return babel.transform(content, {
presets: presets,
comments: false
}).code;
} catch (e) {
console.log('Cannot babelify', path);
console.log(e);
return content;
}
}

function minify_js(content, path) {
try {
return babel.transform(content, {
comments: false
}).code;
} catch (e) {
console.log('Cannot minify', path);
console.log(e);
return content;
}
}

function make_build_map() {
const build_map = {};
for (const app_path of app_paths) {
const build_json_path = p(app_path, 'public', 'build.json');
if (!fs.existsSync(build_json_path)) continue;

let build_json = fs.readFileSync(build_json_path);
try {
build_json = JSON.parse(build_json);
} catch (e) {
console.log(e);
continue;
}

for (const target in build_json) {
const sources = build_json[target];

const new_sources = [];
for (const source of sources) {
const s = p(app_path, source);
new_sources.push(s);
}

if (new_sources.length)
build_json[target] = new_sources;
else
delete build_json[target];
}

Object.assign(build_map, build_json);
}
return build_map;
}

function compile_less() {
return new Promise(function (resolve) {
const promises = [];
for (const app_path of app_paths) {
const public_path = p(app_path, 'public');
const less_path = p(public_path, 'less');
if (!fs.existsSync(less_path)) continue;

const files = fs.readdirSync(less_path);
for (const file of files) {
if(file.includes('variables.less')) continue;
promises.push(compile_less_file(file, less_path, public_path))
}
}

Promise.all(promises).then(() => {
console.log('Less files compiled');
resolve();
});
});
}

function compile_less_file(file, less_path, public_path) {
const file_content = fs.readFileSync(p(less_path, file), 'utf8');
const output_file = file.split('.')[0] + '.css';
console.log('compiling', file);

return less.render(file_content, {
paths: [less_path],
filename: file,
sourceMap: false
}).then(output => {
const out_css = p(public_path, 'css', output_file);
fs.writeFileSync(out_css, output.css);
return out_css;
}).catch(e => {
console.log('Error compiling ', file);
console.log(e);
});
}

function watch_less(ondirty) {
const less_paths = app_paths.map(path => p(path, 'public', 'less'));

const to_watch = [];
for (const less_path of less_paths) {
if (!fs.existsSync(less_path)) continue;
to_watch.push(less_path);
}
chokidar.watch(to_watch).on('change', (filename, stats) => {
console.log(filename, 'dirty');
var last_index = filename.lastIndexOf('/');
const less_path = filename.slice(0, last_index);
const public_path = p(less_path, '..');
filename = filename.split('/').pop();

compile_less_file(filename, less_path, public_path)
.then(css_file_path => {
// build the target css file for which this css file is input
for (const target in build_map) {
const sources = build_map[target];
if (sources.includes(css_file_path)) {
pack(target, sources);
ondirty && ondirty(target);
break;
}
}
})
});
}

function watch_js(ondirty) {
const js_paths = app_paths.map(path => p(path, 'public', 'js'));

const to_watch = [];
for (const js_path of js_paths) {
if (!fs.existsSync(js_path)) continue;
to_watch.push(js_path);
}
chokidar.watch(to_watch).on('change', (filename, stats) => {
console.log(filename, 'dirty');
var last_index = filename.lastIndexOf('/');
const js_path = filename.slice(0, last_index);
const public_path = p(js_path, '..');

// build the target js file for which this js/html file is input
for (const target in build_map) {
const sources = build_map[target];
if (sources.includes(filename)) {
pack(target, sources);
ondirty && ondirty(target);
break;
}
}
});
}

function html_to_js_template(path, content) {
let key = path.split('/');
key = key[key.length - 1];
key = key.split('.')[0];

content = scrub_html_template(content);
return `frappe.templates['${key}'] = '${content}';\n`;
}

function scrub_html_template(content) {
content = content.replace(/\s/g, ' ');
content = content.replace(/(<!--.*?-->)/g, '');
return content.replace("'", "\'");
}

function get_file_size(filepath) {
const stats = fs.statSync(filepath);
const size = stats.size;
// convert it to humanly readable format.
const i = Math.floor(Math.log(size) / Math.log(1024));
return (size / Math.pow(1024, i)).toFixed(2) * 1 + ' ' + ['B', 'KB', 'MB', 'GB', 'TB'][i];
}
20 changes: 17 additions & 3 deletions frappe/build.py
View file
Expand Up @@ -3,6 +3,7 @@

from __future__ import unicode_literals
from frappe.utils.minify import JavascriptMinify
import subprocess

"""
Build the `public` folders and setup languages
Expand All @@ -22,16 +23,30 @@ def setup():
except ImportError: pass
app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]

def bundle(no_compress, make_copy=False, verbose=False):
def bundle(no_compress, make_copy=False, verbose=False, experimental=False):
"""concat / minify js files"""
# build js files
setup()

make_asset_dirs(make_copy=make_copy)

if experimental:
command = 'node ../apps/frappe/frappe/build.js --build'
if not no_compress:
command += ' --minify'
subprocess.call(command.split(' '))
return

build(no_compress, verbose)

def watch(no_compress):
def watch(no_compress, experimental=False):
"""watch and rebuild if necessary"""

if experimental:
command = 'node ../apps/frappe/frappe/build.js --watch'
subprocess.Popen(command.split(' '))
return

setup()

import time
Expand Down Expand Up @@ -101,7 +116,6 @@ def get_build_maps():
except ValueError, e:
print path
print 'JSON syntax error {0}'.format(str(e))

return build_maps

timestamps = {}
Expand Down
10 changes: 6 additions & 4 deletions frappe/commands/utils.py
View file
Expand Up @@ -8,19 +8,21 @@
@click.command('build')
@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
@click.option('--verbose', is_flag=True, default=False, help='Verbose')
def build(make_copy=False, verbose=False):
@click.option('--experimental', is_flag=True, default=False, help='Use the new NodeJS build system')
def build(make_copy=False, verbose=False, experimental=False):
"Minify + concatenate JS and CSS files, build translations"
import frappe.build
import frappe
frappe.init('')
frappe.build.bundle(False, make_copy=make_copy, verbose=verbose)
frappe.build.bundle(False, make_copy=make_copy, verbose=verbose, experimental=experimental)

@click.command('watch')
def watch():
@click.option('--experimental', is_flag=True, default=False, help='Use the new NodeJS build system')
def watch(experimental=False):
"Watch and concatenate JS and CSS files as and when they change"
import frappe.build
frappe.init('')
frappe.build.watch(True)
frappe.build.watch(True, experimental=experimental)

@click.command('clear-cache')
@pass_context
Expand Down

0 comments on commit 743f7ab

Please sign in to comment.