Skip to content
Permalink
Browse files

Release 4.0.5

Release notes are available at:
https://docs.dremio.com/release-notes/40-release-notes.html

Change-Id: I37d1e8e8b9b0fa9671fdf3653946b1f27746d29e
  • Loading branch information
yongyanw committed Nov 20, 2019
1 parent b20ffbf commit 94319dd5d92f488df3290a1eeb7054e9d3155f05
Showing with 4,860 additions and 2,252 deletions.
  1. +1 −1 client/jdbc/pom.xml
  2. +1 −1 client/pom.xml
  3. +1 −1 common/pom.xml
  4. +53 −9 common/src/main/java/com/dremio/common/concurrent/AutoCloseableLock.java
  5. +8 −0 common/src/main/java/com/dremio/common/concurrent/WithAutoCloseableLock.java
  6. +17 −6 common/src/main/java/com/dremio/common/utils/PathUtils.java
  7. +52 −0 common/src/main/java/com/dremio/io/ExponentialBackoff.java
  8. +0 −4 common/src/main/resources/dremio-reference.conf
  9. +1 −1 connector/pom.xml
  10. +1 −1 contrib/hive2-exec-shade/pom.xml
  11. +1 −1 contrib/hive3-exec-shade/pom.xml
  12. +1 −1 contrib/pom.xml
  13. +1 −1 contrib/slim-shade/pom.xml
  14. +1 −1 dac/backend/pom.xml
  15. +1 −1 dac/backend/src/main/java/com/dremio/dac/daemon/DACDaemonModule.java
  16. +31 −16 dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java
  17. +5 −0 dac/backend/src/main/java/com/dremio/dac/model/sources/SourceUI.java
  18. +47 −39 dac/backend/src/main/java/com/dremio/dac/server/doc.md
  19. +2 −1 dac/backend/src/main/java/com/dremio/dac/server/test/TestResource.java
  20. +9 −2 dac/backend/src/main/java/com/dremio/dac/server/tokens/TokenManagerImpl.java
  21. +3 −3 dac/backend/src/main/java/com/dremio/dac/service/catalog/CatalogServiceHelper.java
  22. +1 −1 dac/backend/src/main/java/com/dremio/dac/service/source/SourceService.java
  23. +12 −5 dac/backend/src/test/java/com/dremio/dac/api/TestCatalogResource.java
  24. +1 −3 dac/backend/src/test/java/com/dremio/dac/api/TestPromotion.java
  25. +5 −6 dac/backend/src/test/java/com/dremio/dac/api/TestSourceResource.java
  26. +64 −3 dac/backend/src/test/java/com/dremio/dac/server/TestHomeFiles.java
  27. +2 −0 dac/backend/src/test/java/com/dremio/dac/server/tokens/TestTokenManager.java
  28. +1 −1 dac/common/pom.xml
  29. +4 −4 dac/daemon/pom.xml
  30. +1 −1 dac/daemon/src/main/java/com/dremio/dac/cmd/upgrade/UpdateDatasetSplitIdTask.java
  31. +1 −1 dac/daemon/src/main/java/com/dremio/dac/cmd/upgrade/Upgrade.java
  32. +13 −2 dac/daemon/src/main/java/com/dremio/dac/cmd/upgrade/UpgradeContext.java
  33. +1 −1 dac/daemon/src/test/java/com/dremio/dac/cmd/upgrade/TestUpdatePartitionChunkIdTask.java
  34. +1 −1 dac/daemon/src/test/java/com/dremio/dac/cmd/upgrade/TestUpdateS3CredentialType.java
  35. +1 −1 dac/daemon/src/test/java/com/dremio/dac/cmd/upgrade/TestUpgrade.java
  36. +0 −3 dac/daemon/src/test/java/com/dremio/dac/daemon/TestHdfs.java
  37. +1 −1 dac/pom.xml
  38. +1 −1 dac/ui/pom.xml
  39. +3 −2 dac/ui/src/actions/explore/dataset/data.js
  40. +1 −1 dac/ui/src/components/HeaderItemsTypes/NewQueryButton.js
  41. +3 −2 dac/ui/src/components/Menus/DropdownMenu.js
  42. +2 −7 dac/ui/src/pages/AuthenticationPage/components/LoginForm.js
  43. +2 −2 dac/ui/src/pages/ExplorePage/components/ExploreInfoHeader.js
  44. +23 −13 dac/ui/src/pages/ExplorePage/components/ExploreTable/ExploreTableJobStatus.js
  45. +2 −1 dac/ui/src/pages/ExplorePage/components/ExploreTable/ExploreTableJobStatusSpinner.js
  46. +5 −4 dac/ui/src/pages/ExplorePage/components/TableControls.js
  47. +77 −5 dac/ui/src/reducers/resources/entityReducers/table-spec.js
  48. +28 −11 dac/ui/src/reducers/resources/entityReducers/table.js
  49. +1 −1 dac/ui/src/sagas/performLoadDataset.js
  50. +8 −1 dac/ui/src/selectors/explore.js
  51. +7 −8 distribution/jdbc-driver/pom.xml
  52. +7 −0 distribution/jdbc-driver/src/test/java/com/dremio/jdbc/ITTestShadedJar.java
  53. +2 −1 distribution/jdbc-driver/src/test/java/com/dremio/jdbc/NodeClassLoader.java
  54. +1 −1 distribution/pom.xml
  55. +1 −1 distribution/resources/pom.xml
  56. +16 −16 distribution/resources/src/main/resources/licenses/LICENSES_FOR_DEPENDENCIES.md
  57. +19 −1 distribution/server/pom.xml
  58. +1 −1 plugins/adls/pom.xml
  59. +64 −21 plugins/adls/src/main/java/com/microsoft/azure/datalake/store/AdlsAsyncFileReader.java
  60. +1 −1 plugins/azure/pom.xml
  61. +91 −50 plugins/azure/src/main/java/com/dremio/plugins/azure/AzureStorageFileSystem.java
  62. +1 −1 plugins/elasticsearch/pom.xml
  63. +1 −1 plugins/hive-common/pom.xml
  64. +1 −6 plugins/hive-function-registry/pom.xml
  65. +1 −1 plugins/hive/pom.xml
  66. +6 −6 plugins/hive/src/main/codegen/templates/HiveRecordReaders.java
  67. +15 −13 plugins/hive/src/main/java/com/dremio/exec/store/hive/HiveClient.java
  68. +4 −3 plugins/hive/src/main/java/com/dremio/exec/store/hive/HiveClientWithAuthz.java
  69. +2 −1 plugins/hive/src/main/java/com/dremio/exec/store/hive/HiveStoragePlugin.java
  70. +120 −107 plugins/hive/src/main/java/com/dremio/exec/store/hive/metadata/HiveMetadataUtils.java
  71. +23 −1 plugins/hive/src/main/java/com/dremio/exec/store/hive/orc/ORCFilterPushDownRule.java
  72. +19 −3 plugins/hive/src/main/java/com/dremio/exec/store/hive/orc/ORCSearchArgumentGenerator.java
  73. +1 −0 plugins/hive/src/main/proto/hive.proto
  74. +19 −18 plugins/hive/src/test/java/com/dremio/exec/hive/ITHiveStorage.java
  75. +3 −2 plugins/hive/src/test/java/com/dremio/exec/hive/ITInfoSchemaOnHiveStorage.java
  76. +215 −0 plugins/hive/src/test/java/com/dremio/exec/hive/ITORCFilterPushDown.java
  77. +3 −2 plugins/hive/src/test/java/com/dremio/exec/sql/hive/ITMetadataRefresh.java
  78. +16 −0 plugins/hive/src/test/java/com/dremio/exec/store/hive/HiveTestDataGenerator.java
  79. +1 −1 plugins/hive2/launcher/pom.xml
  80. +7 −2 plugins/hive2/plugin/pom.xml
  81. +1 −1 plugins/hive2/pom.xml
  82. +1 −1 plugins/hive3/launcher/pom.xml
  83. +2 −2 plugins/hive3/plugin/pom.xml
  84. +12 −10 plugins/hive3/plugin/src/main/java/com/dremio/exec/store/hive/HiveClient.java
  85. +120 −107 plugins/hive3/plugin/src/main/java/com/dremio/exec/store/hive/metadata/HiveMetadataUtils.java
  86. +1 −0 plugins/hive3/plugin/src/main/proto/hive.proto
  87. +9 −8 plugins/hive3/plugin/src/test/java/com/dremio/exec/hive/ITHiveStorage.java
  88. +4 −2 plugins/hive3/plugin/src/test/java/com/dremio/exec/hive/ITInfoSchemaOnHiveStorage.java
  89. +0 −3 ...ins/hive3/plugin/src/test/java/com/dremio/exec/impersonation/hive/ITSqlStdBasedAuthorization.java
  90. +0 −2 ...ive3/plugin/src/test/java/com/dremio/exec/impersonation/hive/ITStorageBasedHiveAuthorization.java
  91. +3 −2 plugins/hive3/plugin/src/test/java/com/dremio/exec/sql/hive/ITMetadataRefresh.java
  92. +16 −0 plugins/hive3/plugin/src/test/java/com/dremio/exec/store/hive/HiveTestDataGenerator.java
  93. +1 −1 plugins/hive3/pom.xml
  94. +1 −1 plugins/jdbc/pom.xml
  95. +1 −1 plugins/mongo/pom.xml
  96. +1 −1 plugins/pdfs/pom.xml
  97. +1 −1 plugins/pom.xml
  98. +12 −1 plugins/s3/pom.xml
  99. +2 −2 plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3AsyncByteReader.java
  100. +51 −7 plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3AsyncByteReaderUsingSyncClient.java
  101. +27 −24 plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3FileSystem.java
  102. +10 −5 plugins/s3/src/main/java/com/dremio/plugins/s3/store/STSCredentialProviderV1.java
  103. +105 −6 plugins/s3/src/main/java/com/dremio/plugins/s3/store/STSCredentialProviderV2.java
  104. +58 −0 plugins/s3/src/test/java/com/dremio/plugins/s3/store/TestRetryableInvoker.java
  105. +9 −7 pom.xml
  106. +1 −1 protocol/pom.xml
  107. +1 −1 provision/common/pom.xml
  108. +28 −0 provision/common/src/main/java/com/dremio/provision/service/ProvisioningService.java
  109. +104 −1 provision/common/src/main/java/com/dremio/provision/service/ProvisioningServiceImpl.java
  110. +1 −1 provision/pom.xml
  111. +1 −1 provision/shimloader/pom.xml
  112. +1 −1 provision/yarn/pom.xml
  113. +1 −1 provision/yarn/twill/pom.xml
  114. +1 −1 provision/yarn/yarntwill/pom.xml
  115. +2 −2 sabot/kernel/pom.xml
  116. +4 −4 sabot/kernel/src/main/java/com/dremio/exec/cache/VectorAccessibleSerializable.java
  117. +1 −4 sabot/kernel/src/main/java/com/dremio/exec/catalog/CatalogImpl.java
  118. +86 −523 sabot/kernel/src/main/java/com/dremio/exec/catalog/CatalogServiceImpl.java
  119. +47 −0 sabot/kernel/src/main/java/com/dremio/exec/catalog/CatalogServiceMonitor.java
  120. +7 −5 sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java
  121. +652 −203 sabot/kernel/src/main/java/com/dremio/exec/catalog/ManagedStoragePlugin.java
  122. +2 −24 sabot/kernel/src/main/java/com/dremio/exec/catalog/MetadataSynchronizer.java
  123. +220 −162 sabot/kernel/src/main/java/com/dremio/exec/catalog/PluginsManager.java
  124. +324 −0 sabot/kernel/src/main/java/com/dremio/exec/catalog/SafeNamespaceService.java
  125. +29 −0 sabot/kernel/src/main/java/com/dremio/exec/catalog/SourceAlreadyExistsException.java
  126. +368 −269 sabot/kernel/src/main/java/com/dremio/exec/catalog/SourceMetadataManager.java
  127. +8 −2 sabot/kernel/src/main/java/com/dremio/exec/server/BootStrapContext.java
  128. +1 −1 sabot/kernel/src/main/java/com/dremio/exec/server/SabotNode.java
  129. +0 −19 sabot/kernel/src/main/java/com/dremio/exec/store/CatalogService.java
  130. +4 −4 sabot/kernel/src/main/java/com/dremio/exec/store/dfs/BlockMapBuilder.java
  131. +50 −17 sabot/kernel/src/main/java/com/dremio/exec/store/parquet/Metadata.java
  132. +4 −3 sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetGroupScanUtils.java
  133. +19 −0 sabot/kernel/src/main/java/com/dremio/exec/work/foreman/AttemptManager.java
  134. +4 −0 sabot/kernel/src/main/java/com/dremio/exec/work/foreman/QueryManager.java
  135. +4 −0 sabot/kernel/src/main/java/com/dremio/exec/work/protector/ForemenWorkManager.java
  136. +0 −3 sabot/kernel/src/main/java/com/dremio/sabot/exec/FragmentExecutors.java
  137. +4 −3 sabot/kernel/src/main/java/com/dremio/sabot/exec/FragmentWorkManager.java
  138. +1 −1 sabot/kernel/src/main/java/com/dremio/sabot/rpc/CoordExecService.java
  139. +1 −1 sabot/kernel/src/main/java/com/dremio/sabot/rpc/user/UserServer.java
  140. +2 −2 sabot/kernel/src/test/java/com/dremio/TestExampleQueries.java
  141. +1 −1 sabot/kernel/src/test/java/com/dremio/exec/ExecTest.java
  142. +26 −26 sabot/kernel/src/test/java/com/dremio/exec/catalog/TestCatalogServiceImpl.java
  143. +59 −48 sabot/kernel/src/test/java/com/dremio/exec/catalog/TestFailedToStartPlugin.java
  144. +9 −8 sabot/kernel/src/test/java/com/dremio/exec/catalog/TestPluginsManager.java
  145. +45 −43 sabot/kernel/src/test/java/com/dremio/exec/catalog/TestSourceMetadataManager.java
  146. +0 −3 sabot/kernel/src/test/java/com/dremio/exec/impersonation/TestImpersonationDisabledWithMiniDFS.java
  147. +0 −3 sabot/kernel/src/test/java/com/dremio/exec/impersonation/TestImpersonationQueries.java
  148. +20 −2 sabot/kernel/src/test/java/com/dremio/exec/store/parquet/ParquetRecordReaderTest.java
  149. +1 −1 sabot/logical/pom.xml
  150. +1 −1 sabot/pom.xml
  151. +1 −1 services/accelerator/pom.xml
  152. +27 −2 services/accelerator/src/main/java/com/dremio/service/reflection/ReflectionManager.java
  153. +2 −2 services/base-rpc/pom.xml
  154. +8 −0 services/base-rpc/src/main/java/com/dremio/exec/rpc/ReconnectingConnection.java
  155. +3 −3 services/base-rpc/src/main/java/com/dremio/exec/rpc/RpcBus.java
  156. +7 −1 services/command-pool/pom.xml
  157. +5 −2 services/command-pool/src/main/java/com/dremio/service/commandpool/BoundCommandPool.java
  158. +1 −1 services/configuration/pom.xml
  159. +1 −1 services/coordinator/pom.xml
  160. +24 −0 services/coordinator/src/main/java/com/dremio/service/coordinator/DistributedSemaphore.java
  161. +31 −1 services/coordinator/src/main/java/com/dremio/service/coordinator/local/LocalClusterCoordinator.java
  162. +5 −1 services/coordinator/src/main/java/com/dremio/service/coordinator/zk/ZKClusterClient.java
  163. +72 −2 services/coordinator/src/main/java/com/dremio/service/coordinator/zk/ZkDistributedSemaphore.java
  164. +1 −1 services/datastore/pom.xml
  165. +1 −1 services/datastore/src/main/java/com/dremio/datastore/ByteStoreManager.java
  166. +4 −4 services/datastore/src/main/java/com/dremio/datastore/RemoteKVStore.java
  167. +2 −2 services/datastore/src/main/java/com/dremio/datastore/RocksDBStore.java
  168. +1 −1 services/datastore/src/main/java/com/dremio/datastore/indexed/LuceneSearchIndex.java
  169. +1 −1 services/exec-selector/pom.xml
  170. +1 −1 services/fabric-rpc/pom.xml
  171. +3 −0 services/fabric-rpc/src/main/java/com/dremio/services/fabric/ConnectionManagerRegistry.java
  172. +1 −1 services/jobs/pom.xml
  173. +1 −1 services/namespace/pom.xml
  174. +1 −1 services/options/pom.xml
  175. +3 −2 services/pom.xml
  176. +1 −1 services/resourcescheduler/pom.xml
  177. +1 −1 services/scheduler/pom.xml
  178. +1 −1 services/spill/pom.xml
  179. +18 −7 services/{metrics → telemetry-api}/pom.xml
  180. +159 −0 services/telemetry-api/src/main/java/com/dremio/telemetry/api/Telemetry.java
  181. +143 −0 services/telemetry-api/src/main/java/com/dremio/telemetry/api/config/AutoRefreshConfigurator.java
  182. +1 −1 ...etrics → telemetry-api/src/main/java/com/dremio/telemetry/api/config}/IncludesExcludesFilter.java
  183. +6 −6 ...ator.java → telemetry-api/src/main/java/com/dremio/telemetry/api/config/MetricsConfigurator.java}
  184. +56 −0 services/telemetry-api/src/main/java/com/dremio/telemetry/api/config/RefreshConfiguration.java
  185. +1 −1 .../metrics → telemetry-api/src/main/java/com/dremio/telemetry/api/config}/ReporterConfigurator.java
  186. +62 −0 services/telemetry-api/src/main/java/com/dremio/telemetry/api/config/TelemetryConfigurator.java
  187. +44 −0 services/telemetry-api/src/main/java/com/dremio/telemetry/api/config/TracerConfigurator.java
  188. +1 −1 ...java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/CacheMonitor.java
  189. +1 −1 ...main/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/Counter.java
  190. +1 −1 ...in/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/Histogram.java
  191. +1 −1 ...c/main/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/Meter.java
  192. +10 −66 ...main/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/Metrics.java
  193. +12 −44 ...a/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/ReporterManager.java
  194. +1 −1 ...c/main/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/Timer.java
  195. +1 −1 ...n/java/com/dremio → telemetry-api/src/main/java/com/dremio/telemetry/api}/metrics/TopMonitor.java
  196. +28 −0 services/telemetry-api/src/main/resources/sabot-module.conf
  197. +10 −38 .../com/dremio/metrics → telemetry-api/src/test/java/com/dremio/telemetry/api}/TestMetricConfig.java
  198. +2 −1 ...remio/metrics → telemetry-api/src/test/java/com/dremio/telemetry/api}/TestReportConfigurator.java
  199. +146 −0 ...ces/telemetry-api/src/test/java/com/dremio/telemetry/api/config/TestAutoRefreshConfiguration.java
  200. +1 −1 ...cs → telemetry-api/src/test/java/com/dremio/telemetry/api/config}/TestIncludesExcludesFilter.java
  201. +69 −0 services/telemetry-impl/pom.xml
  202. +102 −0 services/telemetry-impl/src/main/java/com/dremio/telemetry/impl/config/JaegerConfigurator.java
  203. +2 −1 ...mio/metrics → telemetry-impl/src/main/java/com/dremio/telemetry/impl/config}/JmxConfigurator.java
  204. +2 −1 ...o/metrics → telemetry-impl/src/main/java/com/dremio/telemetry/impl/config}/Slf4jConfigurator.java
  205. +1 −2 services/{metrics → telemetry-impl}/src/main/resources/sabot-module.conf
  206. +1 −1 services/users/pom.xml
  207. +1 −1 services/users/src/main/java/com/dremio/service/users/SimpleUserService.java
  208. +1 −1 tools/attach-tool/pom.xml
  209. +1 −1 tools/build-tools/pom.xml
  210. +1 −1 tools/fmpp-maven-plugin/pom.xml
  211. +1 −1 tools/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>com.dremio.client</groupId>
<artifactId>dremio-client-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>
<artifactId>dremio-client-jdbc</artifactId>
<name>Client - JDBC Driver</name>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio</groupId>
<artifactId>dremio-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<groupId>com.dremio.client</groupId>
@@ -23,7 +23,7 @@
<parent>
<groupId>com.dremio</groupId>
<artifactId>dremio-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-common</artifactId>
@@ -15,44 +15,88 @@
*/
package com.dremio.common.concurrent;

import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;

import com.dremio.common.util.Closeable;

import java.util.concurrent.TimeUnit;

/**
* Simple wrapper class that allows Locks to be released via an try-with-resources block.
* Simple wrapper class that allows Locks to be released via an try-with-resources block. Also ensures idempotence.
*/
public class AutoCloseableLock implements Closeable {

private final boolean singleUse;
private final AtomicBoolean closed = new AtomicBoolean(true);
private final Lock lock;

public AutoCloseableLock(Lock lock) {
this(lock, false);
}

public AutoCloseableLock(Lock lock, boolean singleUse) {
this.lock = lock;
this.singleUse = singleUse;
}

public AutoCloseableLock open() {
startOpen();
lock.lock();
return this;
}

private void startOpen() {
if (singleUse && !closed.compareAndSet(true, false)) {
throw new IllegalStateException("Trying to open an already opened lock. AutoCloseableLock allows only one lock count to be held at a time.");
}
}

private boolean shouldClose() {
return !singleUse || closed.compareAndSet(false, true);
}

/**
* Acquires the lock if it is free within the given waiting time and the
* current thread has not been interrupted.
* Acquires the lock if it is free within the given waiting time and the current thread has not been interrupted.
*
* @param time Time to wait.
* @param unit Unit of time.
* @return The AutoCloseableLock instance or null if the wait time expired.
* @return optional AutoCloseableLock
* @throws InterruptedException
*/
public AutoCloseableLock tryOpen(long time, TimeUnit unit) throws InterruptedException {
return lock.tryLock(time, unit) ? this : null;
public Optional<AutoCloseableLock> tryOpen(long time, TimeUnit unit) throws InterruptedException {
startOpen();
if(lock.tryLock(time, unit)) {
return Optional.of(this);
} else {
shouldClose();
return Optional.empty();
}
}

/**
* Close the lock. Will not close the lock if it has already been closed.
*/
@Override
public void close() {
lock.unlock();
if(shouldClose()) {
lock.unlock();
}
}

public static AutoCloseableLock of(Lock lock, boolean singleUse) {
return new AutoCloseableLock(lock, singleUse);
}

public static AutoCloseableLock ofAlreadyOpen(Lock lock, boolean singleUse) {
AutoCloseableLock acl = new AutoCloseableLock(lock, singleUse);
acl.startOpen();
return acl;
}

public static AutoCloseableLock lockAndWrap(Lock lock, boolean singleUse) {
lock.lock();
return ofAlreadyOpen(lock, singleUse);
}

}
@@ -39,6 +39,14 @@ public void close() {
acl.close();
}

/**
* Get the lock that this exposes as an autocloseable.
* @return
*/
public AutoCloseableLock getLock() {
return acl;
}

/**
* Get the value that is included with the lock.
* @return Value of type T.
@@ -278,22 +278,33 @@ public static String relativePath(Path absolutePath, Path basePath) {

/**
* Make sure the <i>givenPath</i> refers to an entity under the given <i>basePath</i>. Idea is to avoid using ".." to
* refer entities outside the ba
* refer entities outside the basePath.
* @param basePath
* @param givenPath
*/
public static void verifyNoAccessOutsideBase(Path basePath, Path givenPath) {
final String basePathNormalized = Path.withoutSchemeAndAuthority(basePath).toString();
final String givenPathNormalized = Path.withoutSchemeAndAuthority(givenPath).toString();
if (!Paths.get(givenPathNormalized).startsWith(Paths.get(basePathNormalized))) {
if (!checkNoAccessOutsideBase(basePath, givenPath)) {
throw UserException.permissionError()
.message("Not allowed to access files outside of the source root")
.addContext("Source root", basePathNormalized)
.addContext("Requested to path", givenPathNormalized)
.addContext("Source root", Path.withoutSchemeAndAuthority(basePath).toString())
.addContext("Requested to path", Path.withoutSchemeAndAuthority(givenPath).toString())
.build(logger);
}
}

/**
* Checks if <i>givenPath</i> refers to an entity under the given <i>basePath</i>. Idea is to avoid using ".." to
* refer entities outside the basePath.
* @param basePath
* @param givenPath
* @return boolean indicating if the givenPath is valid or not relative to the basePath
*/
public static boolean checkNoAccessOutsideBase(Path basePath, Path givenPath) {
final String basePathNormalized = Path.withoutSchemeAndAuthority(basePath).toString();
final String givenPathNormalized = Path.withoutSchemeAndAuthority(givenPath).toString();
return (Paths.get(givenPathNormalized).startsWith(Paths.get(basePathNormalized)));
}

/**
* Remove leading "/"s in the given path string.
* @param path
@@ -0,0 +1,52 @@
/*
* Copyright (C) 2017-2019 Dremio Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.dremio.io;

/**
* This class implements exponential backoff
*/
abstract public class ExponentialBackoff {
/**
* Perform a sleep based on the attempt number, where the sleep period increases exponentially and adds jitter.
*
* @param attemptNumber The retry attempt number, which directly affects the sleep period.
*/
protected final void backoffWait(int attemptNumber) {
try {
// Algorithm taken from https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
// sleep = min(cap, base * 2 ^ attemptNumber)
final double sleep = Math.min(getMaxMillis(), getBaseMillis() * (1 << attemptNumber));
Thread.sleep((long) (Math.random() * (sleep + 1)));
} catch (InterruptedException e) {
// Preserve interrupt status.
Thread.currentThread().interrupt();
}
}

/**
* Return the base millis to sleep as part of exponential backoff.
*
* @return base millis to wait in exponential backoff
*/
protected abstract int getBaseMillis();

/**
* Return the max millis to sleep as part of exponential backoff.
*
* @return max millis to wait in exponential backoff
*/
protected abstract int getMaxMillis();
}
@@ -176,10 +176,6 @@ services: {
# Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}

metrics: {
refresh_millis: 120000
}
}

provisioning: {
@@ -23,7 +23,7 @@
<parent>
<groupId>com.dremio</groupId>
<artifactId>dremio-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-connector</artifactId>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio.contrib</groupId>
<artifactId>dremio-contrib-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-hive2-exec-shaded</artifactId>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio.contrib</groupId>
<artifactId>dremio-contrib-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-hive3-exec-shaded</artifactId>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio</groupId>
<artifactId>dremio-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<groupId>com.dremio.contrib</groupId>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio.contrib</groupId>
<artifactId>dremio-contrib-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-slim-shade</artifactId>
@@ -22,7 +22,7 @@
<parent>
<groupId>com.dremio</groupId>
<artifactId>dremio-dac-parent</artifactId>
<version>4.0.4-201910212053580380-773b665</version>
<version>4.0.5-201911202046080257-19b10938</version>
</parent>

<artifactId>dremio-dac-backend</artifactId>
@@ -157,7 +157,7 @@ public void bootstrap(final Runnable shutdownHook, final SingletonRegistry boots
final boolean isMasterless = config.isMasterlessEnabled();
final boolean embeddedZookeeper = config.getBoolean(DremioConfig.EMBEDDED_MASTER_ZK_ENABLED_BOOL);

bootstrapRegistry.bindSelf(new BootStrapContext(config, scanResult));
bootstrapRegistry.bindSelf(new BootStrapContext(config, scanResult, bootstrapRegistry));

// Start cluster coordinator before all other services so that non master nodes can poll for master status
if (dacConfig.getClusterMode() == ClusterMode.LOCAL) {

0 comments on commit 94319dd

Please sign in to comment.
You can’t perform that action at this time.