Fix rebase issues

trinodb · Aug 19, 2023 · 051a92d · 051a92d
1 parent 0a0356d
commit 051a92d
Show file tree

Hide file tree

Showing 9 changed files with 27 additions and 47 deletions.
diff --git a/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java
@@ -44,7 +44,8 @@ public void testDefaults()
                 .setIncludeCoordinator(true)
                 .setSplitsBalancingPolicy(NodeSchedulerConfig.SplitsBalancingPolicy.STAGE)
                 .setOptimizedLocalScheduling(true)
-                .setAllowedNoMatchingNodePeriod(new Duration(2, MINUTES)));
+                .setAllowedNoMatchingNodePeriod(new Duration(2, MINUTES))
+                .setCacheAffinityPolicy(NONE));
     }
 
     @Test

diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingFileSystemFactory.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingFileSystemFactory.java
@@ -15,14 +15,14 @@
 
 import alluxio.client.file.cache.CacheManager;
 import alluxio.conf.AlluxioConfiguration;
+import com.google.inject.Inject;
 import io.trino.filesystem.TrinoFileSystem;
 import io.trino.filesystem.TrinoFileSystemFactory;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
+import io.trino.hdfs.TrinoHdfsFileSystemStats;
 import io.trino.spi.security.ConnectorIdentity;
 
-import javax.inject.Inject;
-
 import static java.util.Objects.requireNonNull;
 
 public class CachingFileSystemFactory
@@ -31,19 +31,22 @@ public class CachingFileSystemFactory
     private final HdfsEnvironment environment;
     private final CacheManager cacheManager;
     private final AlluxioConfiguration alluxioConf;
+    private final TrinoHdfsFileSystemStats fileSystemStats;
 
     @Inject
-    public CachingFileSystemFactory(HdfsEnvironment environment,
+    public CachingFileSystemFactory(HdfsEnvironment environment, TrinoHdfsFileSystemStats fileSystemStats,
             CacheManager cacheManager, AlluxioConfiguration alluxioConf)
     {
         this.environment = requireNonNull(environment, "environment is null");
+        this.fileSystemStats = requireNonNull(fileSystemStats, "fileSystemStats is null");
         this.cacheManager = requireNonNull(cacheManager, "cacheManager is null");
         this.alluxioConf = requireNonNull(alluxioConf, "alluxioConf is null");
     }
 
     @Override
     public TrinoFileSystem create(ConnectorIdentity identity)
     {
-        return new CachingHdfsFileSystem(environment, new HdfsContext(identity), cacheManager, alluxioConf);
+        return new CachingHdfsFileSystem(environment, new HdfsContext(identity),
+                fileSystemStats, cacheManager, alluxioConf);
     }
 }
diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingHdfsFileSystem.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingHdfsFileSystem.java
@@ -15,32 +15,35 @@
 
 import alluxio.client.file.cache.CacheManager;
 import alluxio.conf.AlluxioConfiguration;
+import io.trino.filesystem.Location;
 import io.trino.filesystem.TrinoInputFile;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
+import io.trino.hdfs.TrinoHdfsFileSystemStats;
 
 public class CachingHdfsFileSystem
         extends HdfsFileSystem
 {
     private final CacheManager cacheManager;
     private final AlluxioConfiguration alluxioConf;
 
-    public CachingHdfsFileSystem(HdfsEnvironment environment, HdfsContext context, CacheManager cacheManager, AlluxioConfiguration alluxioConf)
+    public CachingHdfsFileSystem(HdfsEnvironment environment,
+            HdfsContext context, TrinoHdfsFileSystemStats stats, CacheManager cacheManager, AlluxioConfiguration alluxioConf)
     {
-        super(environment, context);
+        super(environment, context, stats);
         this.cacheManager = cacheManager;
         this.alluxioConf = alluxioConf;
     }
 
     @Override
-    public TrinoInputFile newInputFile(String path)
+    public TrinoInputFile newInputFile(Location location)
     {
-        return new CachingHdfsInputFile(path, null, environment, context, cacheManager, alluxioConf);
+        return new CachingHdfsInputFile(location, null, environment, context, stats.getOpenFileCalls(), cacheManager, alluxioConf);
     }
 
     @Override
-    public TrinoInputFile newInputFile(String path, long length)
+    public TrinoInputFile newInputFile(Location location, long length)
     {
-        return new CachingHdfsInputFile(path, length, environment, context, cacheManager, alluxioConf);
+        return new CachingHdfsInputFile(location, length, environment, context, stats.getOpenFileCalls(), cacheManager, alluxioConf);
     }
 }
diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingHdfsInputFile.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/CachingHdfsInputFile.java
@@ -21,7 +21,9 @@
 import alluxio.hadoop.AlluxioHdfsInputStream;
 import alluxio.hadoop.HdfsFileInputStream;
 import alluxio.wire.FileInfo;
+import io.trino.filesystem.Location;
 import io.trino.filesystem.TrinoInput;
+import io.trino.hdfs.CallStats;
 import io.trino.hdfs.HdfsContext;
 import io.trino.hdfs.HdfsEnvironment;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -40,10 +42,10 @@ public class CachingHdfsInputFile
 
     private final FileSystem.Statistics statistics = new FileSystem.Statistics("alluxio");
 
-    public CachingHdfsInputFile(String path, Long length, HdfsEnvironment environment, HdfsContext context,
-            CacheManager cacheManager, AlluxioConfiguration alluxioConf)
+    public CachingHdfsInputFile(Location location, Long length, HdfsEnvironment environment, HdfsContext context,
+            CallStats stats, CacheManager cacheManager, AlluxioConfiguration alluxioConf)
     {
-        super(path, length, environment, context);
+        super(location, length, environment, context, stats);
         this.cacheManager = cacheManager;
         this.alluxioConf = alluxioConf;
     }

diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystem.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystem.java
@@ -50,7 +50,7 @@ public class HdfsFileSystem
 {
     protected final HdfsEnvironment environment;
     protected final HdfsContext context;
-    private final TrinoHdfsFileSystemStats stats;
+    protected final TrinoHdfsFileSystemStats stats;
 
     private final Map<FileSystem, Boolean> hierarchicalFileSystemCache = new IdentityHashMap<>();
 

diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsInputFile.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsInputFile.java
@@ -36,6 +36,7 @@
 class HdfsInputFile
         implements TrinoInputFile
 {
+    protected final Location location;
     protected final HdfsEnvironment environment;
     protected final HdfsContext context;
     protected final Path file;

diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/InternalHiveConnectorFactory.java
@@ -26,9 +26,6 @@
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.trace.Tracer;
 import io.trino.filesystem.manager.FileSystemModule;
-import io.trino.filesystem.hdfs.HdfsFileSystemModule;
-import io.trino.filesystem.hdfs.cache.CachingFileSystemConfig;
-import io.trino.filesystem.hdfs.cache.CachingFileSystemModule;
 import io.trino.hdfs.HdfsModule;
 import io.trino.hdfs.authentication.HdfsAuthenticationModule;
 import io.trino.hdfs.azure.HiveAzureModule;

diff --git a/pom.xml b/pom.xml
@@ -1575,7 +1575,7 @@
             <dependency>
                 <groupId>org.alluxio</groupId>
                 <artifactId>alluxio-shaded-client</artifactId>
-                <version>2.9.3</version>
+                <version>301-SNAPSHOT</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.slf4j</groupId>

diff --git a/testing/trino-server-dev/etc/config.properties b/testing/trino-server-dev/etc/config.properties
@@ -28,35 +28,8 @@ query.client.timeout=5m
 query.min-expire-age=30m
 
 plugin.bundles=\
-  ../../plugin/trino-resource-group-managers/pom.xml,\
-  ../../plugin/trino-password-authenticators/pom.xml, \
-  ../../plugin/trino-iceberg/pom.xml,\
-  ../../plugin/trino-delta-lake/pom.xml,\
-  ../../plugin/trino-blackhole/pom.xml,\
-  ../../plugin/trino-cassandra/pom.xml,\
-  ../../plugin/trino-memory/pom.xml,\
   ../../plugin/trino-jmx/pom.xml,\
-  ../../plugin/trino-raptor-legacy/pom.xml,\
   ../../plugin/trino-hive-hadoop2/pom.xml,\
-  ../../plugin/trino-hudi/pom.xml,\
-  ../../plugin/trino-example-http/pom.xml,\
-  ../../plugin/trino-kafka/pom.xml, \
-  ../../plugin/trino-tpch/pom.xml, \
-  ../../plugin/trino-local-file/pom.xml, \
-  ../../plugin/trino-mysql/pom.xml,\
-  ../../plugin/trino-mariadb/pom.xml,\
-  ../../plugin/trino-singlestore/pom.xml,\
-  ../../plugin/trino-sqlserver/pom.xml, \
-  ../../plugin/trino-prometheus/pom.xml, \
-  ../../plugin/trino-postgresql/pom.xml, \
-  ../../plugin/trino-thrift/pom.xml, \
-  ../../plugin/trino-tpcds/pom.xml, \
-  ../../plugin/trino-google-sheets/pom.xml, \
-  ../../plugin/trino-druid/pom.xml, \
-  ../../plugin/trino-geospatial/pom.xml, \
-  ../../plugin/trino-http-event-listener/pom.xml, \
-  ../../plugin/trino-exchange-filesystem/pom.xml, \
-  ../../plugin/trino-exchange-hdfs/pom.xml, \
-  ../../plugin/trino-mysql-event-listener/pom.xml
+  ../../plugin/trino-tpcds/pom.xml
 
 node-scheduler.include-coordinator=true