Skip to content

[common] Load plugin classes directly from jar without decompression #1996

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 2 commits into from
Sep 15, 2023
Merged

[common] Load plugin classes directly from jar without decompression #1996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a828c6
[common] Load plugin classes directly from jar without decompression
tsreaper Sep 15, 2023
764ea04
Remove useless try catch
tsreaper Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions paimon-codegen-loader/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,10 @@ under the License.
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-table-planner-jars</id>
<id>copy-codegen-classes</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
Expand All @@ -78,10 +78,9 @@ under the License.
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>paimon-codegen.jar</destFileName>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
<outputDirectory>${project.build.directory}/classes/paimon-codegen</outputDirectory>
</configuration>
</execution>
</executions>
Expand Down
4 changes: 2 additions & 2 deletions paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
/** Loader to load codegen classes. */
public class CodeGenLoader {

private static final String CODEGEN_FAT_JAR = "paimon-codegen.jar";
private static final String CODEGEN_CLASSES_DIR = "paimon-codegen";

// Singleton lazy initialization

Expand All @@ -32,7 +32,7 @@ public class CodeGenLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
loader = new PluginLoader(CODEGEN_FAT_JAR);
loader = new PluginLoader(CODEGEN_CLASSES_DIR);
}
return loader;
}
Expand Down
86 changes: 12 additions & 74 deletions paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,11 @@

package org.apache.paimon.plugin;

import org.apache.paimon.utils.FileIOUtils;
import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.LocalFileUtils;
import org.apache.paimon.utils.StringUtils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.ServiceLoader;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -62,54 +49,22 @@ public class PluginLoader {

private static final String[] COMPONENT_CLASSPATH = new String[] {"org.apache.paimon"};

private final Path delegateJar;

private final ComponentClassLoader submoduleClassLoader;

public PluginLoader(String jarName) {
try {
ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
Path tmpDirectory =
tmpDirectoryFromYarn()
.orElseGet(() -> Paths.get(System.getProperty("java.io.tmpdir")));
Files.createDirectories(
LocalFileUtils.getTargetPathIfContainsSymbolicPath(tmpDirectory));
delegateJar =
extractResource(
jarName,
ownerClassLoader,
tmpDirectory,
String.format(
"%s could not be found.\n"
+ "If you're running a test, please make sure you've built the modules by running\n"
+ "mvn clean install -DskipTests",
jarName));
this.submoduleClassLoader =
new ComponentClassLoader(
new URL[] {delegateJar.toUri().toURL()},
ownerClassLoader,
OWNER_CLASSPATH,
COMPONENT_CLASSPATH);
} catch (IOException e) {
throw new RuntimeException("Could not initialize the paimon-codegen loader.", e);
public PluginLoader(String dirName) {
// URL must end with slash,
// otherwise URLClassLoader cannot recognize this path as a directory
if (!dirName.endsWith("/")) {
dirName += "/";
}
}

private Path extractResource(
String resourceName,
ClassLoader flinkClassLoader,
Path tmpDirectory,
String errorMessage)
throws IOException {
String[] splitName = resourceName.split("\\.");
splitName[0] += "_" + UUID.randomUUID();
final Path tempFile = Files.createFile(tmpDirectory.resolve(String.join(".", splitName)));
final InputStream resourceStream = flinkClassLoader.getResourceAsStream(resourceName);
if (resourceStream == null) {
throw new RuntimeException(errorMessage);
}
IOUtils.copyBytes(resourceStream, Files.newOutputStream(tempFile));
return tempFile;
ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
this.submoduleClassLoader =
new ComponentClassLoader(
new URL[] {ownerClassLoader.getResource(dirName)},
ownerClassLoader,
OWNER_CLASSPATH,
COMPONENT_CLASSPATH);
}

public <T> T discover(Class<T> clazz) {
Expand Down Expand Up @@ -141,21 +96,4 @@ public <T> T newInstance(String name) {
public ClassLoader submoduleClassLoader() {
return submoduleClassLoader;
}

@Override
protected void finalize() throws IOException {
submoduleClassLoader.close();
FileIOUtils.deleteFileOrDirectory(delegateJar.toFile());
}

private static Optional<Path> tmpDirectoryFromYarn() {
String localDirs = System.getenv("LOCAL_DIRS");
if (!StringUtils.isBlank(localDirs)) {
String[] paths = localDirs.split(",|" + File.pathSeparator);
if (paths.length > 0) {
return Optional.of(Paths.get(paths[0]));
}
}
return Optional.empty();
}
}
7 changes: 3 additions & 4 deletions paimon-filesystems/paimon-oss/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,10 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-oss-jar</id>
<id>copy-oss-classes</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
Expand All @@ -74,10 +74,9 @@
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>paimon-plugin-oss.jar</destFileName>
<outputDirectory>${project.build.directory}/classes/paimon-plugin-oss</outputDirectory>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
</configuration>
</execution>
</executions>
Expand Down
4 changes: 2 additions & 2 deletions paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
/** A {@link PluginLoader} to load oss. */
public class OSSLoader implements FileIOLoader {

private static final String OSS_JAR = "paimon-plugin-oss.jar";
private static final String OSS_CLASSES_DIR = "paimon-plugin-oss";

private static final String OSS_CLASS = "org.apache.paimon.oss.OSSFileIO";

Expand All @@ -42,7 +42,7 @@ public class OSSLoader implements FileIOLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
loader = new PluginLoader(OSS_JAR);
loader = new PluginLoader(OSS_CLASSES_DIR);
}
return loader;
}
Expand Down
7 changes: 3 additions & 4 deletions paimon-filesystems/paimon-s3/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,10 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-s3-jar</id>
<id>copy-s3-classes</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
Expand All @@ -112,10 +112,9 @@
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>paimon-plugin-s3.jar</destFileName>
<outputDirectory>${project.build.directory}/classes/paimon-plugin-s3</outputDirectory>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
</configuration>
</execution>
</executions>
Expand Down
4 changes: 2 additions & 2 deletions paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
/** A {@link PluginLoader} to load oss. */
public class S3Loader implements FileIOLoader {

private static final String S3_JAR = "paimon-plugin-s3.jar";
private static final String S3_CLASSES_DIR = "paimon-plugin-s3";

private static final String S3_CLASS = "org.apache.paimon.s3.S3FileIO";

Expand All @@ -42,7 +42,7 @@ public class S3Loader implements FileIOLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
loader = new PluginLoader(S3_JAR);
loader = new PluginLoader(S3_CLASSES_DIR);
}
return loader;
}
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ under the License.

<properties>
<project.build.outputTimestamp>2023-09-06T02:16:30Z</project.build.outputTimestamp>
<paimon.shade.version>0.4.0-incubating</paimon.shade.version>
<paimon.shade.version>0.6-SNAPSHOT</paimon.shade.version>
<paimon.shade.jackson.version>2.14.2</paimon.shade.jackson.version>
<paimon.shade.guava.version>30.1.1-jre</paimon.shade.guava.version>
<paimon.shade.caffeine.version>2.9.3</paimon.shade.caffeine.version>
Expand Down