Add SpectrumPluginPlatform

Summary:
@public

This diff adds a new plugin which exposes the platform decoder on Android (i.e. `BitmapFactory`) as decompressors for various image formats to Spectrum. The benefit is that one e.g. no longer needs to ship the WebP plugin if you are just interested in decoding WebP files. Also it newly adds support for GIF and HEIC files on Android versions that support it. However, note that we are crossing the JNI barrier a bit more often and allocate an additional (temporary) Bitmap which can be a bottleneck.

Before explaining the underlying structure, be warned. The changes are a bit intricate and cover: Java life cycle, JNI calls in both directions, reflection, C++ iterators, RAII bitmap locks, and more. Hope you enjoy it.

How does this work:

 - The `JniSpectrumPluginPlatform` creates a new Plugin which references the `JniPlatformDecompressor` for various file formats – i.e. it creates multiple `DecompressorFactory` instances. The `JniPlatformDecompressor` implements our usual `IDecompressor` interface but is agnostic to the incoming file format.
 - When being instantiated, it will create an instance of the `SpectrumPlatformDecompressor` in the Java world and pass the entire `IImageSource` content as a `byte[]` array
 - When the interface methods are called, it will lazily request the image specification or Bitmap data and hold a reference locally for subsequent calls.
   - The `image::Specification` is derived from the `Bitmap#Options` object in Java
   - The `image::Scanline` is copied out of the temporary locked `JBitmap` object using a RAII helper
 - Tests are using our normal declarative testing framework

Missing bits:

 - Test with HEIC files (probably not to be included as automated tests as only supported on physical devices)
 - Support for sampling (at least the JPEG 1/2, 1/4, 1/8 ones)
 - Optimization: using the `RegionDecoder` to have less allocations

Reviewed By: cuva

Differential Revision: D15758468

fbshipit-source-id: c1ae9f3cdd8dba0943ee1dcdc638201f95beb2f6

android/spectrumpluginplatform/CMakeLists.txt

@@ -0,0 +1,46 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+cmake_minimum_required (VERSION 3.6.0)
+project(spectrum CXX)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_VERBOSE_MAKEFILE on)
+
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Os")
+
+set(spectrumpluginplatform_DIR ${CMAKE_CURRENT_LIST_DIR}/src/main/cpp)
+file(GLOB spectrumpluginplatform_SOURCES
+  ${spectrumpluginplatform_DIR}/spectrumjni/plugins/JniSpectrumPluginPlatform.cpp
+  ${spectrumpluginplatform_DIR}/spectrumjni/plugins/JniSpectrumPlatformDecompressor.cpp
+  ${spectrumpluginplatform_DIR}/spectrumjni/OnLoad.cpp
+)
+
+add_library(spectrumpluginplatform SHARED
+  ${spectrumpluginplatform_SOURCES}
+)
+
+target_compile_options(spectrumpluginplatform PRIVATE
+  -DSPECTRUM_OSS
+  -fexceptions
+)
+
+target_include_directories(spectrumpluginplatform PRIVATE
+  ${spectrumpluginplatform_DIR}
+)
+
+set(EXTERNAL_DIR ${CMAKE_CURRENT_LIST_DIR}/../../androidLibs/third-party/)
+set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
+file(MAKE_DIRECTORY ${BUILD_DIR})
+
+set(spectrum_DIR ${CMAKE_CURRENT_LIST_DIR}/../)
+set(spectrum_BUILD_DIR ${BUILD_DIR}/spectrumjni/${ANDROID_ABI})
+add_subdirectory(${spectrum_DIR} ${spectrum_BUILD_DIR})
+
+target_link_libraries(spectrumpluginplatform
+  spectrumfbjni
+  spectrumcpp
+  spectrum
+)

android/spectrumpluginplatform/build.gradle

@@ -0,0 +1,64 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+apply plugin: 'com.android.library'
+apply plugin: 'maven'
+
+android {
+    compileSdkVersion rootProject.compileSdkVersion
+    buildToolsVersion rootProject.buildToolsVersion
+
+    defaultConfig {
+        minSdkVersion rootProject.minSdkVersion
+        targetSdkVersion rootProject.targetSdkVersion
+        buildConfigField "boolean", "IS_INTERNAL_BUILD", 'true'
+
+        ndk {
+            abiFilters 'x86', 'armeabi-v7a', 'x86_64', 'arm64-v8a'
+        }
+
+        externalNativeBuild {
+            cmake {
+                arguments '-DANDROID_TOOLCHAIN=clang', '-DANDROID_STL=c++_shared'
+                targets 'spectrumpluginplatform'
+            }
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path './CMakeLists.txt'
+        }
+    }
+
+    packagingOptions {
+        // provided by the main spectrum target
+        exclude "**/libc++_shared.so"
+        exclude "**/libspectrumcpp.so"
+        exclude "**/libspectrumfbjni.so"
+        exclude "**/libspectrum.so"
+    }
+}
+
+dependencies {
+    implementation project(':android')
+    implementation project(':fbjni')
+    compileOnly deps.jsr305
+    implementation deps.soloader
+
+    testImplementation deps.festAssert
+    testImplementation deps.junit
+    testImplementation deps.mockitoCore
+    testImplementation deps.robolectric
+}
+
+apply from: rootProject.file('gradle/release.gradle')
+
+task sourcesJar(type: Jar) {
+    from android.sourceSets.main.java.srcDirs
+    classifier = 'sources'
+}
+
+artifacts.add('archives', sourcesJar)

android/spectrumpluginplatform/gradle.properties

@@ -0,0 +1,9 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+POM_NAME=Spectrum Platform plugin
+POM_DESCRIPTION=Spectrum Platform plugin for Android
+POM_ARTIFACT_ID=spectrum-platform
+POM_PACKAGING=aar

android/spectrumpluginplatform/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.facebook.spectrum.plugins.platform">
+</manifest>

android/spectrumpluginplatform/src/main/cpp/spectrumjni/OnLoad.cpp

@@ -0,0 +1,14 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include "spectrumjni/plugins/JniSpectrumPluginPlatform.h"
+
+#include <fbjni/fbjni.h>
+
+jint JNI_OnLoad(JavaVM* vm, void*) {
+  return facebook::jni::initialize(vm, [] {
+    facebook::spectrum::plugins::JSpectrumPluginPlatform::registerNatives();
+  });
+}

android/spectrumpluginplatform/src/main/cpp/spectrumjni/plugins/JniSpectrumPlatformDecompressor.cpp

@@ -0,0 +1,125 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include "JniSpectrumPlatformDecompressor.h"
+
+#include <spectrum/core/Constants.h>
+#include <spectrumjni/image/JniSpecification.h>
+#include <spectrumjni/BitmapPixelsLock.h>
+
+namespace facebook {
+namespace spectrum {
+namespace plugins {
+namespace platform {
+
+//
+// JSpectrumPlatformDecompressor
+//
+
+image::Specification JSpectrumPlatformDecompressor::getImageSpecification() {
+    static const auto method =
+            javaClassStatic()->getMethod<facebook::jni::local_ref<image::JSpecification>()>(
+                    "getImageSpecification");
+    return method(self())->toNative();
+}
+
+facebook::jni::local_ref<jni::JBitmap> JSpectrumPlatformDecompressor::readBitmap() {
+    static const auto method =
+            javaClassStatic()->getMethod<facebook::jni::local_ref<jni::JBitmap>()>(
+                    "readBitmap");
+    return method(self());
+}
+
+facebook::jni::local_ref<JSpectrumPlatformDecompressor>
+JSpectrumPlatformDecompressor::make(const std::vector<std::uint8_t> &content) {
+    auto jByteArray = facebook::jni::JArrayByte::newArray(content.size());
+
+    // copy read bytes to java heap from source: for this we reinterpret the content vector as
+    // signed chars (Java's byte type) which have the same underlying byte size
+    static_assert(
+            sizeof(signed char) == sizeof(std::uint8_t),
+            "wtf signed char and std::uint8_t differ in size");
+    jByteArray->setRegion(0, content.size(), reinterpret_cast<const signed char *>(content.data()));
+
+    return JSpectrumPlatformDecompressor::newInstance(jByteArray);
+}
+
+//
+// JniPlatformDecompressor
+//
+
+JniPlatformDecompressor::JniPlatformDecompressor(
+        io::IImageSource &source,
+        const folly::Optional <image::Ratio> &samplingRatio) {
+    std::vector<std::uint8_t> content{};
+    content.reserve(source.available());
+
+    std::vector<std::uint8_t> buffer(core::DefaultBufferSize);
+    std::size_t bytesRead;
+    while ((bytesRead = source.read(reinterpret_cast<char *const>(buffer.data()), buffer.size())) > 0) {
+        content.insert(content.end(), buffer.begin(), buffer.begin() + bytesRead);
+    }
+
+    _jSpectrumPlatformDecompressor = JSpectrumPlatformDecompressor::make(content);
+}
+
+image::Specification JniPlatformDecompressor::sourceImageSpecification() {
+    _ensureImageSpecificationRead();
+    return *_imageSpecification;
+}
+
+image::Specification JniPlatformDecompressor::outputImageSpecification() {
+    return sourceImageSpecification();
+}
+
+std::unique_ptr<image::Scanline> JniPlatformDecompressor::readScanline() {
+    _ensureImageSpecificationRead();
+    _ensureBitmapRead();
+    if (_outputScanline >= _imageSpecification->size.height) {
+        return nullptr;
+    }
+
+    jni::BitmapPixelsLock bmpLock(
+            facebook::jni::Environment::current(), _jBitmap.get());
+    const uint8_t* pixelPtr = bmpLock.getPixelsPtr();
+
+    SPECTRUM_ERROR_CSTR_IF(
+            pixelPtr == nullptr,
+            codecs::error::DecompressorFailure,
+            "failed_to_lock_bitmap");
+
+    const auto& pixelSpec = _imageSpecification->pixelSpecification;
+    const auto width = _imageSpecification->size.width;
+    auto scanline = std::make_unique<image::Scanline>(pixelSpec, width);
+
+    SPECTRUM_ERROR_CSTR_IF(
+            scanline->sizeBytes() != bmpLock.getScanlineSizeBytes(),
+            codecs::error::DecompressorFailure,
+            "unexpected_input_scanline_size");
+
+    // copy data from bitmap to scanline
+    const std::size_t offset = _outputScanline * scanline->sizeBytes();
+    memcpy(scanline->data(), pixelPtr + offset, scanline->sizeBytes());
+    ++_outputScanline;
+
+    return scanline;
+}
+
+void JniPlatformDecompressor::_ensureBitmapRead() {
+    if (!_jBitmap) {
+        _jBitmap = _jSpectrumPlatformDecompressor->readBitmap();
+    }
+}
+
+void JniPlatformDecompressor::_ensureImageSpecificationRead() {
+    if (!_imageSpecification.has_value()) {
+        _imageSpecification = _jSpectrumPlatformDecompressor->getImageSpecification();
+    }
+}
+
+} // namespace platform
+} // namespace plugins
+} // namespace spectrum
+} // namespace facebook

android/spectrumpluginplatform/src/main/cpp/spectrumjni/plugins/JniSpectrumPlatformDecompressor.h

@@ -0,0 +1,65 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <spectrum/io/IEncodedImageSource.h>
+#include <spectrum/io/IImageSource.h>
+#include <spectrum/image/Specification.h>
+#include <spectrum/codecs/IDecompressor.h>
+#include <spectrumjni/JniBaseTypes.h>
+
+#include <cstddef>
+#include <exception>
+
+namespace facebook {
+namespace spectrum {
+namespace plugins {
+namespace platform {
+
+/**
+* Wrapping class for the SpectrumPlatformDecompressor.
+*/
+class JSpectrumPlatformDecompressor : public facebook::jni::JavaClass<JSpectrumPlatformDecompressor> {
+public:
+    static constexpr const char* kJavaDescriptor = "Lcom/facebook/spectrum/plugins/SpectrumPlatformDecompressor;";
+    image::Specification getImageSpecification();
+    facebook::jni::local_ref<jni::JBitmap> readBitmap();
+
+    static facebook::jni::local_ref<JSpectrumPlatformDecompressor> make(const std::vector<std::uint8_t>& content);
+};
+
+/**
+ * Providing the IDecompressor implementation using the JSpectrumPlatformDecompressor to be used by the native plugin.
+ */
+class JniPlatformDecompressor :  public codecs::IDecompressor {
+private:
+    facebook::jni::local_ref<JSpectrumPlatformDecompressor> _jSpectrumPlatformDecompressor;
+    facebook::jni::local_ref<jni::JBitmap> _jBitmap;
+
+    folly::Optional<image::Specification> _imageSpecification;
+    std::size_t _outputScanline = 0;
+
+public:
+    explicit JniPlatformDecompressor(
+            io::IImageSource& source,
+            const folly::Optional<image::Ratio>& samplingRatio);
+
+    ~JniPlatformDecompressor() override = default;
+
+    image::Specification sourceImageSpecification() override;
+    image::Specification outputImageSpecification() override;
+    std::unique_ptr<image::Scanline> readScanline() override;
+
+private:
+    void _ensureImageSpecificationRead();
+    void _ensureBitmapRead();
+};
+
+} // namespace platform
+} // namespace plugins
+} // namespace spectrum
+} // namespace facebook

android/spectrumpluginplatform/src/main/cpp/spectrumjni/plugins/JniSpectrumPluginPlatform.cpp

@@ -0,0 +1,54 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include "JniSpectrumPluginPlatform.h"
+#include "JniSpectrumPlatformDecompressor.h"
+
+#include <spectrum/image/Specification.h>
+#include <spectrum/Plugin.h>
+
+namespace facebook {
+namespace spectrum {
+namespace plugins {
+
+codecs::DecompressorProvider makeLibPlatformDecompressorProvider(const image::EncodedFormat& format) {
+  return {
+          .format = format,
+          .supportedSamplingRatios = {},
+          .decompressorFactory = [](io::IImageSource& source,
+                                    const folly::Optional<image::Ratio>& samplingRatio,
+                                    const Configuration& /* unused */) {
+              return std::make_unique<platform::JniPlatformDecompressor>(source, samplingRatio);
+          },
+  };
+}
+
+jlong JSpectrumPluginPlatform::nativeCreatePlugin() {
+  const auto plugin = new Plugin{};
+  plugin->decompressorProviders.push_back(makeLibPlatformDecompressorProvider(image::formats::Gif));
+  plugin->decompressorProviders.push_back(makeLibPlatformDecompressorProvider(image::formats::Heif));
+  plugin->decompressorProviders.push_back(makeLibPlatformDecompressorProvider(image::formats::Jpeg));
+  plugin->decompressorProviders.push_back(makeLibPlatformDecompressorProvider(image::formats::Png));
+  plugin->decompressorProviders.push_back(makeLibPlatformDecompressorProvider(image::formats::Webp));
+
+  static_assert(sizeof(void*) <= sizeof(jlong), "sizeof(void*) <= sizeof(jlong)");
+  return reinterpret_cast<jlong>(plugin);
+}
+
+facebook::jni::local_ref<JSpectrumPluginPlatform::jhybriddata>
+JSpectrumPluginPlatform::initHybrid(facebook::jni::alias_ref<jclass>) {
+  return makeCxxInstance();
+}
+
+void JSpectrumPluginPlatform::registerNatives() {
+  registerHybrid(
+      {makeNativeMethod(
+           "nativeCreatePlugin", JSpectrumPluginPlatform::nativeCreatePlugin),
+       makeNativeMethod("initHybrid", JSpectrumPluginPlatform::initHybrid)});
+}
+
+} // namespace plugins
+} // namespace spectrum
+} // namespace facebook