LPS-83379 Enabled by default

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/bnd.bnd

@@ -0,0 +1,3 @@
+Bundle-Name: Liferay Document Library Asset Auto Tagger Tensorflow
+Bundle-SymbolicName: com.liferay.document.library.asset.auto.tagger.tensorflow
+Bundle-Version: 1.0.0

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/build.gradle

@@ -0,0 +1,49 @@
+import de.undercouch.gradle.tasks.download.Download
+
+apply plugin: "de.undercouch.download"
+
+task copyInceptionModel(type: Copy)
+task downloadInceptionModel(type: Download)
+
+sourceCompatibility = "1.8"
+targetCompatibility = "1.8"
+
+classes {
+	dependsOn copyInceptionModel
+}
+
+copyInceptionModel {
+	dependsOn downloadInceptionModel
+	from {
+		zipTree(downloadInceptionModel.dest)
+	}
+
+	includeEmptyDirs = false
+	into "classes/META-INF/model"
+}
+
+dependencies {
+	compileInclude group: "org.tensorflow", name: "libtensorflow", version: "1.8.0"
+	compileInclude group: "org.tensorflow", name: "libtensorflow_jni", version: "1.8.0"
+	compileInclude group: "org.tensorflow", name: "tensorflow", version: "1.8.0"
+
+	compileOnly group: "biz.aQute.bnd", name: "biz.aQute.bndlib", version: "3.1.0"
+	compileOnly group: "com.liferay.portal", name: "com.liferay.portal.impl", version: "default"
+	compileOnly group: "com.liferay.portal", name: "com.liferay.portal.kernel", version: "default"
+	compileOnly group: "de.undercouch", name: "gradle-download-task", version: "3.2.0"
+	compileOnly group: "org.osgi", name: "org.osgi.core", version: "6.0.0"
+	compileOnly group: "org.osgi", name: "org.osgi.service.component.annotations", version: "1.3.0"
+	compileOnly project(":apps:asset:asset-auto-tagger-api")
+	compileOnly project(":apps:static:portal-configuration:portal-configuration-metatype-api")
+	compileOnly project(":core:petra:petra-concurrent")
+	compileOnly project(":core:petra:petra-process")
+	compileOnly project(":core:petra:petra-reflect")
+	compileOnly project(":core:petra:petra-string")
+}
+
+downloadInceptionModel {
+	dest new File(buildDir, "inception5h.zip")
+	onlyIfNewer true
+	overwrite false
+	src "https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip"
+}

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/src/main/java/com/liferay/document/library/asset/auto/tagger/tensorflow/internal/TensorFlowImageAssetAutoTagProvider.java

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+package com.liferay.document.library.asset.auto.tagger.tensorflow.internal;
+
+import com.liferay.asset.auto.tagger.AssetAutoTagProvider;
+import com.liferay.document.library.asset.auto.tagger.tensorflow.internal.configuration.TensorflowImageAssetAutoTagProviderConfiguration;
+import com.liferay.document.library.asset.auto.tagger.tensorflow.internal.util.InceptionImageLabeler;
+import com.liferay.portal.configuration.metatype.bnd.util.ConfigurableUtil;
+import com.liferay.portal.kernel.exception.PortalException;
+import com.liferay.portal.kernel.log.Log;
+import com.liferay.portal.kernel.log.LogFactoryUtil;
+import com.liferay.portal.kernel.repository.capabilities.TemporaryFileEntriesCapability;
+import com.liferay.portal.kernel.repository.model.FileEntry;
+import com.liferay.portal.kernel.repository.model.FileVersion;
+import com.liferay.portal.kernel.util.ArrayUtil;
+import com.liferay.portal.kernel.util.ContentTypes;
+import com.liferay.portal.kernel.util.FileUtil;
+
+import java.io.IOException;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Modified;
+import org.osgi.service.component.annotations.Reference;
+
+/**
+ * @author Alejandro Tardín
+ */
+@Component(
+	configurationPid = "com.liferay.document.library.asset.auto.tagger.tensorflow.internal.configuration.TensorflowImageAssetAutoTagProviderConfiguration",
+	property = "model.class.name=com.liferay.document.library.kernel.model.DLFileEntry",
+	service = AssetAutoTagProvider.class
+)
+public class TensorFlowImageAssetAutoTagProvider
+	implements AssetAutoTagProvider<FileEntry> {
+
+	@Override
+	public List<String> getTagNames(FileEntry fileEntry) {
+		if (_tensorflowImageAutoTaggerConfiguration.enabled() &&
+			!_isTemporary(fileEntry)) {
+
+			try {
+				FileVersion fileVersion = fileEntry.getFileVersion();
+
+				if (_accepts(fileVersion.getMimeType())) {
+					return _inceptionImageLabeler.label(
+						FileUtil.getBytes(fileVersion.getContentStream(false)),
+						_tensorflowImageAutoTaggerConfiguration.
+							confidenceThreshold());
+				}
+			}
+			catch (IOException | PortalException e) {
+				_log.error(e, e);
+			}
+		}
+
+		return Collections.emptyList();
+	}
+
+	@Activate
+	@Modified
+	protected void activate(Map<String, Object> properties) {
+		_tensorflowImageAutoTaggerConfiguration =
+			ConfigurableUtil.createConfigurable(
+				TensorflowImageAssetAutoTagProviderConfiguration.class,
+				properties);
+	}
+
+	private boolean _accepts(String mimeType) {
+		return ArrayUtil.contains(_SUPPORTED_MIME_TYPES, mimeType);
+	}
+
+	private boolean _isTemporary(FileEntry fileEntry) {
+		return fileEntry.isRepositoryCapabilityProvided(
+			TemporaryFileEntriesCapability.class);
+	}
+
+	private static final String[] _SUPPORTED_MIME_TYPES = {
+		ContentTypes.IMAGE_JPEG, ContentTypes.IMAGE_BMP, ContentTypes.IMAGE_PNG
+	};
+
+	private static final Log _log = LogFactoryUtil.getLog(
+		TensorFlowImageAssetAutoTagProvider.class);
+
+	@Reference
+	private InceptionImageLabeler _inceptionImageLabeler;
+
+	private volatile TensorflowImageAssetAutoTagProviderConfiguration
+		_tensorflowImageAutoTaggerConfiguration;
+
+}

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/src/main/java/com/liferay/document/library/asset/auto/tagger/tensorflow/internal/configuration/TensorflowImageAssetAutoTagProviderConfiguration.java

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+package com.liferay.document.library.asset.auto.tagger.tensorflow.internal.configuration;
+
+import aQute.bnd.annotation.metatype.Meta;
+
+import com.liferay.portal.configuration.metatype.annotations.ExtendedObjectClassDefinition;
+
+/**
+ * @author Alejandro Tardín
+ */
+@ExtendedObjectClassDefinition(category = "documents-and-media")
+@Meta.OCD(
+	id = "com.liferay.document.library.asset.auto.tagger.tensorflow.internal.configuration.TensorflowImageAssetAutoTagProviderConfiguration",
+	localization = "content/Language",
+	name = "tensorflow-auto-tag-provider-configuration-name"
+)
+public interface TensorflowImageAssetAutoTagProviderConfiguration {
+
+	/**
+	 * Enables auto tagging of images using a pre-trained tensorflow model
+	 */
+	@Meta.AD(
+		deflt = "true", description = "enabled-description", name = "enabled",
+		required = false
+	)
+	public boolean enabled();
+
+	/**
+	 * Sets the confidenceThreshold for the returned tags
+	 */
+	@Meta.AD(
+		deflt = "0.1", description = "confidence-threshold-description",
+		name = "confidence-threshold", required = false
+	)
+	public float confidenceThreshold();
+
+}

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/src/main/java/com/liferay/document/library/asset/auto/tagger/tensorflow/internal/process/GetLabelProbabilitiesProcessCallable.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+package com.liferay.document.library.asset.auto.tagger.tensorflow.internal.process;
+
+import com.liferay.petra.process.ProcessCallable;
+import com.liferay.petra.process.ProcessException;
+
+/**
+ * @author Shuyang Zhou
+ */
+public class GetLabelProbabilitiesProcessCallable
+	implements ProcessCallable<float[]> {
+
+	public GetLabelProbabilitiesProcessCallable(byte[] imageBytes) {
+		_imageBytes = imageBytes;
+	}
+
+	@Override
+	public float[] call() throws ProcessException {
+		return InceptionImageLabelerUtil.getLabelProbabilities(_imageBytes);
+	}
+
+	private static final long serialVersionUID = 1L;
+
+	private final byte[] _imageBytes;
+
+}

modules/apps/document-library/document-library-asset-auto-tagger-tensorflow/src/main/java/com/liferay/document/library/asset/auto/tagger/tensorflow/internal/process/InceptionImageLabelerUtil.java

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ */
+
+package com.liferay.document.library.asset.auto.tagger.tensorflow.internal.process;
+
+import com.liferay.document.library.asset.auto.tagger.tensorflow.internal.util.GraphBuilder;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.tensorflow.Graph;
+import org.tensorflow.Output;
+import org.tensorflow.Session;
+import org.tensorflow.Tensor;
+
+/**
+ * Based on https://github.com/tensorflow/tensorflow/blob/master/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java
+ *
+ * @author Shuyang Zhou
+ * @author Alejandro Tardín
+ */
+public class InceptionImageLabelerUtil {
+
+	public static float[] getLabelProbabilities(byte[] imageBytes) {
+		try (Tensor<Float> imageTensor = _normalizeImage(imageBytes);
+			Tensor<Float> resultTensor = _getOutputTensor(
+				_imageLabelerGraph, imageTensor)) {
+
+			long[] shape = resultTensor.shape();
+
+			if ((resultTensor.numDimensions() != 2) || (shape[0] != 1)) {
+				throw new RuntimeException(
+					String.format(
+						"Expected model to produce a [1 N] shaped tensor " +
+							"where N is the number of labels, instead it " +
+								"produced one with shape %s",
+						Arrays.toString(shape)));
+			}
+
+			int numberOfLabels = (int)shape[1];
+
+			return resultTensor.copyTo(new float[1][numberOfLabels])[0];
+		}
+	}
+
+	private static Tensor<Float> _getOutputTensor(
+		Graph graph, Tensor<Float> inputTensor) {
+
+		try (Session session = new Session(graph)) {
+			Session.Runner runner = session.runner();
+
+			runner = runner.feed("input", inputTensor);
+			runner = runner.fetch("output");
+
+			List<Tensor<?>> tensors = runner.run();
+
+			Tensor<?> resultTensor = tensors.get(0);
+
+			return resultTensor.expect(Float.class);
+		}
+	}
+
+	private static Tensor<Float> _normalizeImage(byte[] imageBytes) {
+		try (Tensor tensor = Tensor.create(imageBytes, String.class)) {
+			return _getOutputTensor(_imageNormalizerGraph, tensor);
+		}
+	}
+
+	private static final Graph _imageLabelerGraph;
+	private static final Graph _imageNormalizerGraph;
+
+	static {
+		try (InputStream inputStream =
+				InceptionImageLabelerUtil.class.getResourceAsStream(
+					"/META-INF/model/tensorflow_inception_graph.pb")) {
+
+			byte[] buffer = new byte[1024];
+
+			try (ByteArrayOutputStream byteArrayOutputStream =
+					new ByteArrayOutputStream()) {
+
+				int size = -1;
+
+				while ((size = inputStream.read(buffer)) != -1) {
+					byteArrayOutputStream.write(buffer, 0, size);
+				}
+
+				_imageLabelerGraph = new Graph();
+
+				_imageLabelerGraph.importGraphDef(
+					byteArrayOutputStream.toByteArray());
+			}
+
+			_imageNormalizerGraph = new Graph();
+
+			GraphBuilder builder = new GraphBuilder(_imageNormalizerGraph);
+
+			Output<String> input = builder.placeholder("input", String.class);
+
+			builder.rename(
+				builder.div(
+					builder.sub(
+						builder.resizeBilinear(
+							builder.expandDims(
+								builder.cast(
+									builder.decodeJpeg(input, 3), Float.class),
+								builder.constant("make_batch", 0)),
+							builder.constant("size", new int[] {224, 224})),
+						builder.constant("mean", 117F)),
+					builder.constant("scale", 1F)),
+				"output"
+			);
+		}
+		catch (IOException ioe) {
+			throw new ExceptionInInitializerError(ioe);
+		}
+	}
+
+}