Skip to content

Commit

Permalink
Fix webpack builds. Support uploading images from a directory.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeresig
jeresig committed Sep 8, 2018
1 parent e9d7804 commit 26d6513
Show file tree
Hide file tree
Showing 9 changed files with 190 additions and 37 deletions.
28 changes: 21 additions & 7 deletions src/bin/mongaku.js
View file
Expand Up @@ -61,7 +61,9 @@ if (args.v || args.version) {
const srcDir = path.join(rootDir, "src");
const buildDir = path.join(rootDir, "build");

shell.exec(`${getBinary("babel")} ${srcDir} --out-dir ${buildDir}`);
shell.exec(
`${getBinary("babel")} ${srcDir} --out-dir ${buildDir} --verbose`,
);

const webpackConfig = path.join(rootDir, "webpack.config.js");
shell.exec(`${getBinary("webpack")} --config ${webpackConfig}`);
Expand All @@ -70,10 +72,17 @@ if (args.v || args.version) {
const srcDir = path.join(rootDir, "src");
const buildDir = path.join(rootDir, "build");

shell.exec(`${getBinary("babel")} ${srcDir} --out-dir ${buildDir} -w`);

const webpackConfig = path.join(rootDir, "webpack.config.js");
shell.exec(`${getBinary("webpack")} --config ${webpackConfig} -w`);
console.log(
shell.exec(`${getBinary("webpack")} --config ${webpackConfig} -w`, {
async: true,
}),
);

shell.exec(
`${getBinary("babel")} ${srcDir} --out-dir ${buildDir} -w --verbose`,
{async: true},
);
} else if (cmd === "dev") {
const cwd = process.cwd();
const localDir = localFile("..");
Expand All @@ -96,16 +105,21 @@ if (args.v || args.version) {
.concat(extraArgs)
.join(" ");

shell.exec(devCmd);
shell.exec(devCmd, {async: true});

const rootDir = localFile("../..");
const srcDir = path.join(rootDir, "src");
const buildDir = path.join(rootDir, "build");

shell.exec(`${getBinary("babel")} ${srcDir} --out-dir ${buildDir} -w`);
shell.exec(
`${getBinary("babel")} ${srcDir} --out-dir ${buildDir} -w --verbose`,
{async: true},
);

const webpackConfig = path.join(rootDir, "webpack.config.js");
shell.exec(`${getBinary("webpack")} --config ${webpackConfig} -w`);
shell.exec(`${getBinary("webpack")} --config ${webpackConfig} -w`, {
async: true,
});
} else if (cmd === "create" || cmd === "convert" || cmd === "i18n") {
const [name] = extraArgs;

Expand Down
2 changes: 2 additions & 0 deletions src/lib/default-record-options.js
View file
Expand Up @@ -6,6 +6,8 @@ module.exports = {
imagesRequired: true,
noImages: false,
noImageSearch: false,
allowDirectoryUpload: false,
autoID: false,

name: i18n => i18n.gettext("Records"),

Expand Down
49 changes: 46 additions & 3 deletions src/logic/admin.js
View file
Expand Up @@ -196,7 +196,7 @@ module.exports = function(app) {
}
},

uploadImages(req, res, next) {
uploadZipFile(req, res, next) {
const {source, i18n, lang} = req;

const form = new formidable.IncomingForm();
Expand Down Expand Up @@ -237,6 +237,42 @@ module.exports = function(app) {
});
},

uploadDirectory(req, res, next) {
const {source, i18n, lang} = req;

const form = new formidable.IncomingForm();
form.encoding = "utf-8";

form.parse(req, (err, {directory}) => {
/* istanbul ignore if */
if (err) {
return next(
new Error(i18n.gettext("Error processing directory.")),
);
}

if (!directory) {
return next(
new Error(i18n.gettext("No directory specified.")),
);
}

const batch = ImageImport.fromFile(directory, source._id);
batch.directory = directory;

batch.save(err => {
/* istanbul ignore if */
if (err) {
return next(
new Error(i18n.gettext("Error saving directory.")),
);
}

res.redirect(source.getAdminURL(lang));
});
});
},

uploadData(req, res, next) {
const {source, i18n, lang} = req;

Expand Down Expand Up @@ -318,11 +354,18 @@ module.exports = function(app) {
this.admin,
);
app.post(
"/:type/source/:source/upload-images",
"/:type/source/:source/upload-zip",
auth,
canEdit,
source,
this.uploadZipFile,
);
app.post(
"/:type/source/:source/upload-directory",
auth,
canEdit,
source,
this.uploadImages,
this.uploadDirectory,
);
app.post(
"/:type/source/:source/upload-data",
Expand Down
70 changes: 51 additions & 19 deletions src/schemas/ImageImport.js
View file
Expand Up @@ -2,6 +2,7 @@ const os = require("os");
const fs = require("fs");
const path = require("path");

const glob = require("glob");
const async = require("async");
const unzip = require("unzip-stream");

Expand All @@ -18,7 +19,13 @@ const states = [
id: "started",
name: i18n => i18n.gettext("Awaiting processing..."),
advance(batch, callback) {
batch.processImages(callback);
if (batch.zipFile) {
batch.processZipFile(callback);
} else if (batch.directory) {
batch.processDirectory(callback);
} else {
process.nextTick(callback);
}
},
},
{
Expand Down Expand Up @@ -69,7 +76,11 @@ const ImageImport = new db.schema(
// (temporary, deleted after processing)
zipFile: {
type: String,
required: true,
},

// The directory location of the source images to import
directory: {
type: String,
},

// The name of the original file (e.g. `foo.zip`)
Expand Down Expand Up @@ -97,7 +108,7 @@ Object.assign(ImageImport.methods, Import.methods, {
return states;
},

processImages(callback) {
processZipFile(callback) {
const zipFile = fs.createReadStream(this.zipFile);
let zipError;
const files = [];
Expand Down Expand Up @@ -153,26 +164,47 @@ Object.assign(ImageImport.methods, Import.methods, {
return callback(new Error("ZIP_FILE_EMPTY"));
}

// Import all of the files as images
async.eachLimit(
files,
1,
(file, callback) => {
this.addResult(file, callback);
},
err => {
/* istanbul ignore if */
if (err) {
return callback(err);
}

this.setSimilarityState(callback);
},
);
this.importImages(files, callback);
});
});
},

processDirectory(callback) {
const fileGlob = path.join(
path.resolve(this.directory),
"**",
"*.@(jpg|jpeg)",
);

glob(fileGlob, (err, files) => {
/* istanbul ignore if */
if (err) {
return callback(err);
}

this.importImages(files, callback);
});
},

importImages(files, callback) {
// Import all of the files as images
async.eachLimit(
files,
1,
(file, callback) => {
this.addResult(file, callback);
},
err => {
/* istanbul ignore if */
if (err) {
return callback(err);
}

this.setSimilarityState(callback);
},
);
},

setSimilarityState(callback) {
const {type} = this.getSource();
const hasImageSearch = options.types[type].hasImageSearch();
Expand Down
2 changes: 1 addition & 1 deletion src/tests/init.js
View file
Expand Up @@ -5,7 +5,7 @@ const tap = require("tap");
const sinon = require("sinon");
const mockfs = require("mock-fs");
const async = require("async");
const iconv = require("iconv-lite");
const iconv = require("body-parser/node_modules/iconv-lite");

// Force ICONV to pre-load its encodings
iconv.getCodec("utf8");
Expand Down
59 changes: 55 additions & 4 deletions src/views/Admin.js
View file
Expand Up @@ -111,9 +111,7 @@ const UploadImagesForm = ({source}: Props, {gettext, URL}: Context) => (
</div>
<div className="panel-body">
<form
action={URL(
`/${source.type}/source/${source._id}/upload-images`,
)}
action={URL(`/${source.type}/source/${source._id}/upload-zip`)}
method="POST"
encType="multipart/form-data"
>
Expand Down Expand Up @@ -156,6 +154,57 @@ const UploadImagesForm = ({source}: Props, {gettext, URL}: Context) => (

UploadImagesForm.contextTypes = childContextTypes;

const UploadDirectoryForm = ({source}: Props, {gettext, URL}: Context) => (
<div className="panel panel-default">
<div className="panel-heading">
<h3 className="panel-title">
{gettext("Upload Directory of Images")}
</h3>
</div>
<div className="panel-body">
<form
action={URL(
`/${source.type}/source/${source._id}/upload-directory`,
)}
method="POST"
encType="multipart/form-data"
>
<p>
{gettext(
"Upload a directory of " +
"JPG images (.jpg or .jpeg).",
)}{" "}
{gettext(
"Names of images should match " +
"the names provided in the metadata.",
)}{" "}
{gettext(
"Directory must be a file path on the same system" +
" on which this server is running.",
)}
</p>

<div className="form-inline">
<div className="form-group">
<input
type="text"
name="directory"
className="form-control"
/>
</div>{" "}
<input
type="submit"
value={gettext("Upload")}
className="btn btn-primary"
/>
</div>
</form>
</div>
</div>
);

UploadDirectoryForm.contextTypes = childContextTypes;

const DataImport = (
{batch}: Props & {batch: Import},
{gettext, format, fixedDate}: Context,
Expand Down Expand Up @@ -276,12 +325,14 @@ UploadDataForm.contextTypes = childContextTypes;

const Admin = (props: Props, {options}: Context) => {
const {title, imageImport, dataImport, source} = props;
const hasImages = options.types[source.type].hasImages;
const {hasImages, allowDirectoryUpload} = options.types[source.type];

return (
<div>
<h1>{title}</h1>
{hasImages && <UploadImagesForm {...props} />}
{hasImages &&
allowDirectoryUpload && <UploadDirectoryForm {...props} />}
{imageImport.length > 0 && <ImageImports {...props} />}
<UploadDataForm {...props} />
{dataImport.length > 0 && <DataImports {...props} />}
Expand Down
3 changes: 2 additions & 1 deletion src/views/types.js
View file
Expand Up @@ -120,7 +120,8 @@ type TypeOptions = {
hasImages: boolean,
requiresImages: boolean,
hasImageSearch: boolean,
autoID?: boolean,
allowDirectoryUpload: boolean,
autoID: boolean,
};

export type Source = {
Expand Down
12 changes: 10 additions & 2 deletions src/views/utils.js
View file
Expand Up @@ -41,11 +41,19 @@ module.exports = (

// Format a number using commas
stringNum(num: number): string {
return numberFormat.format(num);
try {
return numberFormat.format(num);
} catch (e) {
return num.toString();
}
},

fixedDate(date: Date): string {
return dateFormat.format(date);
try {
return dateFormat.format(date);
} catch (e) {
return date.toString();
}
},

format(fmt: string = "", props: {[key: string]: any}): string {
Expand Down
2 changes: 2 additions & 0 deletions webpack.config.js
View file
Expand Up @@ -27,6 +27,8 @@ for (const file of entries) {
module.exports = {
entry,

mode: process.env.NODE_ENV === "production" ? "production" : "development",

output: {
filename: "[name].js",
path: path.resolve(process.cwd(), "static", "js"),
Expand Down

0 comments on commit 26d6513

Please sign in to comment.