feat: batch implementation preparation

Author: piercus

README.md

@@ -3,3 +3,10 @@
 This library has been freely inspired from [imgaug](https://github.com/aleju/imgaug)
 
 It is made to work with [hasard](https://www.npmjs.com/package/hasard) and [opencv4nodejs](https://github.com/justadudewhohacks/opencv4nodejs)
+
+## Todo list
+
+[] Add benchmark test to measure the speed
+[] Faster random generator using [tensorflow js truncated normal](https://js.tensorflow.org/api/1.0.0/#truncatedNormal)
+[] Run augmentation by batches on tensorflowjs
+[x] Get affine and perspective transform to work with tensorflow backend

lib/augmenters/abstract.js

@@ -12,29 +12,29 @@ const img = cv.imread('lenna.png')
 const augmenter = new ia.PerspectiveTransform(0.2)
 * @example
 // Run the augmenter once
-const {image} = augmenter.runAugmenter(img)
-cv.imread('output.png', image);
+const {images} = augmenter.runAugmenter([img])
+cv.imwrite('output.png', images[0]);
 * @example
-// Run the augmenter multiple times
-augmenter.run({image, times: 10});
-// => [{ image : Mat }, { image : Mat }, ...]
+// Run the augmenter 4 times
+augmenter.run({images: [img, img, img, img]});
+// => { images : [Mat, Mat, ...]}
 * @example
 // follow a point in the augmentation workflow
-augmenter.run({image, points: [[25, 90], [12, 32]]})
-// => [{ image : Mat, points: [<first point new position>, <second point new position>] }]
+augmenter.run({images: [img], points: [[[25, 90], [12, 32]]]})
+// => { images : [Mat], points: [[<first point new position>, <second point new position>]] }
 **/
 class AbstractAugmenter extends Abstract{
 	constructor(opts) {
 		super(opts)
 		this._augmenter = true;
 	}
 	/**
-	* @typedef {AugmenterFormat | Image} OneRunOption
+	* @typedef {AugmenterFormat | Images} OneRunOption
 	**/
 
 	/**
 	* @typedef {Object} AugmenterFormat
-	* @property {Image} image the image to augment
+	* @property {Images} images the image to augment
 	* @property {Array.<Point>} points the points to augment
 	* @property {Array.<Box>} boxes bournding boxes to augment
 	*/
@@ -46,33 +46,41 @@ class AbstractAugmenter extends Abstract{
 	*/
 	runAugmenter(runOpts) {
 		let params1;
-		if (runOpts && runOpts.image && this.backend.isImage(runOpts.image)) {
+		if (runOpts && runOpts.images && this.backend.isImages(runOpts.images)) {
 			params1 = runOpts;
-		} else if (this.backend.isImage(runOpts)) {
-			params1 = {image: runOpts};
+		} else if (this.backend.isImages(runOpts)) {
+			params1 = {images: runOpts};
 		} else {
-			console.log(runOpts.image, this.backend.key)
-			throw(new Error('runOnce must have an image in it'));
+			throw(new Error('runOnce must have images in it'));
 		}
-		const metadata = this.backend.getMetadata(params1.image);
-
-		const points = (params1.points || []).map(p => {
-			if (Array.isArray(p)) {
-				return this.backend.point(p[0], p[1]);
-			}
-			return p;
-		});
+		const metadata = this.backend.getMetadata(params1.images);
+		
 		debug(`buildHasard ${this._name}`);
 
-		const o2 = Object.assign({}, {boxes: []}, metadata, params1, {points});
+		const o2 = Object.assign({}, metadata, params1);
+		const nImages = metadata.nImages;
+
 		const params = this.buildHasard(o2);
+		
 		debug(`runOnce ${this._name}`);
-		const resolved = hasard.isHasard(params) ? params.runOnce(runOpts) : params;
+		const resolved = [];
+		
+		// every image hasard is generated independantly
+		for(var i = 0; i< nImages; i++){
+			const resolvedParams = hasard.isHasard(params) ? params.runOnce(runOpts) : params
+			resolved.push(resolvedParams);
+		}
 		//console.log({resolved, params})
 		debug(`augment ${this._name}`);
 		return Promise.resolve(this.augment(o2, resolved, runOpts));
 	}
 
+	fromFilenames({filenames}){
+		return this.backend.readImages(filenames).then(images => {
+			this.runAugmenter({images: images})
+		});
+	}
+	
 	buildHasard(o){
 		return this.buildParams(o);
 	}
@@ -98,19 +106,51 @@ class AbstractAugmenter extends Abstract{
 	// 		return this.runAugmenter();
 	// 	}
 	// }
-	augment(attr, opts) {
-		this.checkParams(opts);
+	augment(attrs, opts, runOpts) {
+		opts.forEach(o => {
+			this.checkParams(o);
+		})
+		
+		const res = this.backend.splitImages(attrs.images).map((image, index) => {
+			const points = ((attrs.points && attrs.points[index]) || []).map(p => {
+				if (Array.isArray(p)) {
+					return this.backend.point(p[0], p[1]);
+				}
+				return p;
+			});
+			
+			const newAttrs = Object.assign(
+				{}, 
+				attrs, 
+				{image}, {images: null},
+				{points},
+				{boxes: (attrs.boxes && attrs.boxes[index]) || []}
+			)			
+			return this.augmentOne(
+				newAttrs, 
+				opts[index],
+				runOpts
+			)
+		});
+		
 		return {
-			image: this.augmentImage(attr, opts),
-			boxes: this.augmentBoxes(attr, opts),
-			points: this.augmentPoints(attr, opts)
+			images: res.map(r => r.image),
+			boxes: res.map(r => r.boxes),
+			points: res.map(r => r.points)
 		};
 	}
-
-	augmentImage({image}) {
+	augmentOne(attr, opts, runOpts) {
+		return {
+			image: this.augmentImage(attr, opts, runOpts),
+			boxes: this.augmentBoxes(attr, opts, runOpts),
+			points: this.augmentPoints(attr, opts, runOpts)
+		};
+	}
+	
+	augmentImage({images}, opts, runOpts) {
+		// by default do nothing
 		return image;
 	}
-
 	augmentPoints({points}) {
 		return points;
 	}

lib/augmenters/add-weighted.js

@@ -5,10 +5,10 @@ const AbstractAugmenter = require('./abstract');
 /**
 * Adds noise sampled from a gaussian distribution
 * @param {Object} opts options
-* @param {Array.<Number>} opts.value length-3 RGB pixel to add to the image
+* @param {Array.<Number>} opts.value length-3 RGB pixel to add to the images
 * @param {Number} [opts.alpha=1] 
 * @example
-// Simple usage, add 12 red to the image's pixels
+// Simple usage, overlay 10% opacity white layer over the image
 new ia.AddWeighted({
 	value: [255, 255, 255],
 	alpha: 0.1

lib/augmenters/crop.js

@@ -63,7 +63,7 @@ class CropAugmenter extends AbstractAugmenter {
 
 
 	getRect({width, height, params}) {
-		const size = params.size;
+		const size = params.size.concat();
 		return {
 			x: size[0],
 			y: size[1],
@@ -80,7 +80,7 @@ class CropAugmenter extends AbstractAugmenter {
 		return this.backend.crop(image, rect);
 	}
 
-	augmentPoints({points, width, height}, params) {
+	augmentPointsOnImage({points, width, height}, params) {
 		const rect = this.getRect({width, height, params});
 		const origin = this.backend.point(rect.x, rect.y);
 		return points.map(p => p.sub(origin));

lib/augmenters/sequential.js

@@ -28,21 +28,34 @@ class Sequential extends AbstractAugmenter {
 		return this.steps;
 	}
 	augment(o, steps, runOpts) {
-		let promise = Promise.resolve(o)
-		steps.forEach(s => {
-			s.setBackend(this.backend.key)
-			//console.log({s})
-			promise = promise.then(current => {
-				if(!this.backend.isImage(current.image)){
-					throw(new Error(`Can only augment on object with image property in it`))
-				}
-				if(this.backend.isEmptyImage(current.image)){
-					throw(new Error(`Empty image not allowed`))
-				}
-				return s.runAugmenter(Object.assign({},runOpts, current));
-			})
+		const promises = this.backend.splitImages(o.images).map((image, imageIndex) => {
+			let promise = Promise.resolve({images: [image]})
+			steps[imageIndex].forEach(s => {
+				s.setBackend(this.backend.key)
+				//console.log({s})
+				promise = promise.then(current => {
+
+					if(!this.backend.isImages(current.images)){
+						//console.log(this.backend.isImages.toString(), current)
+						throw(new Error(`Can only augment on object with image property in it`))
+					}
+					if(this.backend.isEmptyImages(current.images)){
+						throw(new Error(`Empty image not allowed`))
+					}
+					const runOpts2 = Object.assign({},runOpts, current);
+					return s.runAugmenter(runOpts2);
+				})
+			});
+			return promise;
+		})
+		
+		return Promise.all(promises).then(results => {
+			return {
+				images: results.map(({images}) => images).reduce((a,b) => a.concat(b)), 
+				points: results.map(({points}) => points).reduce((a,b) => a.concat(b)),
+				boxes: results.map(({boxes}) => boxes).reduce((a,b) => a.concat(b))
+			}
 		});
-		return promise;
 	}
 }
 module.exports = Sequential;

lib/backend/index.js

@@ -16,8 +16,12 @@ module.exports = {
 	get(k){
 		if(k === 'opencv4nodejs'){
 			return new OpenCVBackend();
-		} else if('tfjs'){
+		} else if(k === 'tfjs'){
 			return new TensorflowBackend()
+		} else if(k.version && typeof(k.version.tfjs) === 'string'){
+			return new TensorflowBackend(k)
+		} else {
+			throw(new Error(`invalid backend ${k}`))
 		}
 	}
 };

lib/backend/opencv-backend.js

@@ -1,5 +1,15 @@
 const debug = require('debug')('image-augment:backend:opencv4nodejs');
 
+const pluralAny = function(fn){
+	return function(imgs){
+		if(Array.isArray(imgs)){
+			return imgs.map(fn).filter(a => !a).length === 0
+		} else {
+			return false
+		}
+	}
+}
+
 class OpenCVBackend {
 	constructor() {
 		this._cv = require('opencv4nodejs');
@@ -40,17 +50,22 @@ class OpenCVBackend {
 		}
 		return false
 	}
+	isImages(imgs){
+		return pluralAny(this.isImage)(imgs)
+	}
 	isEmptyImage(img){
 		if(img.cols === 0 || img.rows === 0){
 			return true
 		}
 		return false
 	}
+	isEmptyImages(imgs){
+		return pluralAny(this.isEmptyImage)(imgs)
+	}
 	resize(image, width, height){
 		return image.resize(height, width);
 	}
 	drawBoxes(image, boxes, color, thickness = 1){
-		console.log({boxes})
 		boxes.forEach(box => {
 			image.drawRectangle(
 				this.point(box[0], box[1]), 
@@ -106,6 +121,15 @@ class OpenCVBackend {
 		return resImg;
 	}
 
+	addWeighteds(imgs, imgs2, alphas) {
+		if(imgs.length !== imgs2.length || imgs.length !== alpha.length){
+			throw(new Error('lenght does not match'))
+		}
+		return imgs.map((img, i) => {
+			return this.addWeighted(imgs[i], imgs2[i], alphas[i]);
+		})
+	}
+	
 	addWeighted(img, img2, alpha) {
 		const typeSigned = img.channels === 4 ? this._cv.CV_16SC4 : this._cv.CV_16SC3;
 		const typeFinal = img.channels === 4 ? this._cv.CV_8UC4 : this._cv.CV_8UC3;
@@ -116,7 +140,6 @@ class OpenCVBackend {
 		const resImg = added.convertTo(typeFinal);
 		return resImg;
 	}
-
 	blur(img, size) {
 		return img.blur(new this._cv.Size(size[0], size[1]));
 	}
@@ -209,7 +232,9 @@ class OpenCVBackend {
 	readImage(filename) {
 		return this._cv.imreadAsync(filename, this._cv.IMREAD_UNCHANGED);
 	}
-
+	readImages(filenames) {
+		return Promise.all(filenames.map(filename => this.readImage(filename)));
+	}
 	overlay({foreground, background, width, height, channels}) {
 
 		if (channels === 3) {
@@ -243,16 +268,29 @@ class OpenCVBackend {
 	writeImage(filename, img) {
 		return this._cv.imwriteAsync(filename, img);
 	}
-
+	writeImages(filenames, imgs) {
+		if(imgs.length !== filenames.length){
+			throw (new Error('array length and filenames lenght should match'))
+		}
+		return Promise.all(imgs.map((img, i) => this.writeImage(filenames[i], img)))
+	}
 	imageToBuffer(img) {
-		return img.getData();
+		return Promise.resolve(img.getData());
 	}
-
-	getMetadata(img) {
+	imagesToBuffer(imgs) {
+		return Promise.all(imgs.map(this.imageToBuffer)).then(buffers => {
+			return Buffer.concat(buffers);
+		})	
+	}
+	splitImages(imgs){
+		return imgs;
+	}
+	getMetadata(imgs) {
 		return {
-			width: img.cols,
-			height: img.rows,
-			channels: img.channels
+			nImages: imgs.length,
+			width: imgs[0].cols,
+			height: imgs[0].rows,
+			channels: imgs[0].channels
 		};
 	}