Permalink
Browse files

Merge trunk into the HDFS-347 branch.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-347@1467511 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
2 parents 999f6d1 + 74ab6d5 commit 673a7b06b8b978746d920e62ddd366b6eddaa1a2 @atm atm committed Apr 12, 2013
Showing with 13,172 additions and 2,738 deletions.
  1. +0 −4 hadoop-client/pom.xml
  2. +45 −2 hadoop-common-project/hadoop-common/CHANGES.txt
  3. +1 −1 hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
  4. +4 −20 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
  5. +0 −4 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
  6. +65 −0 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
  7. +1 −1 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsFilter.java
  8. +6 −1 ...ommon-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
  9. +22 −6 ...c/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
  10. +18 −0 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
  11. +1 −1 hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
  12. +6 −2 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
  13. +0 −36 ...n-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
  14. +524 −0 ...n-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java
  15. +70 −0 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
  16. +101 −0 ...project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java
  17. +316 −0 .../hadoop-common/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java
  18. +362 −0 ...adoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
  19. +14 −2 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
  20. +63 −0 ...mmon-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/filter/TestPatternFilter.java
  21. +38 −13 ...-project/hadoop-common}/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
  22. +32 −0 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
  23. +28 −0 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MockitoUtil.java
  24. +61 −0 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
  25. +16 −1 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
  26. +4 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
  27. +1 −1 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
  28. +13 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
  29. +3 −1 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
  30. +15 −1 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
  31. +2 −1 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
  32. +15 −0 ...src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
  33. +14 −0 ...doop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
  34. +17 −0 .../src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
  35. +5 −2 ...project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
  36. +1 −1 ...fs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
  37. +1 −0 ...hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
  38. +1 −1 ...hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
  39. +3 −3 ...fs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
  40. +259 −0 ...ain/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
  41. +4 −0 ...op-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
  42. +3 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
  43. +30 −5 ...dfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
  44. +54 −0 ...op-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
  45. +23 −8 ...-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
  46. +9 −3 ...roject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
  47. +174 −141 ...p-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
  48. +6 −1 ...s-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
  49. +33 −16 ...fs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
  50. +8 −1 ...hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
  51. +3 −1 ...oject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
  52. +2 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/GSet.java
  53. +5 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/GSetByHashMap.java
  54. +12 −8 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java
  55. +2 −9 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
  56. +10 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
  57. +28 −0 hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
  58. +23 −0 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
  59. +1 −0 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
  60. +9 −9 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
  61. +35 −0 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetTimes.java
  62. +18 −7 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
  63. +26 −10 ...ct/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
  64. +6 −0 ...-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
  65. +1 −1 ...roject/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolManager.java
  66. +303 −0 ...java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestAvailableSpaceVolumeChoosingPolicy.java
  67. +15 −6 ...est/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
  68. +2 −1 ...dfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
  69. +1 −14 ...hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
  70. +29 −15 ...-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
  71. +69 −0 ...ct/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
  72. +5 −0 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java
  73. +50 −0 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
  74. BIN hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
  75. +15 −1 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
  76. +216 −37 hadoop-mapreduce-project/CHANGES.txt
  77. +138 −68 ...ent/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
  78. +2 −1 ...p-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/JobStateInternal.java
  79. +2 −2 ...mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java
  80. +23 −4 ...ava/org/apache/hadoop/mapreduce/v2/app/{recover/package-info.java → job/event/JobStartEvent.java}
  81. +1 −0 ...e-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptEventType.java
  82. +50 −0 ...lient-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptRecoverEvent.java
  83. +1 −0 ...apreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskEventType.java
  84. +50 −0 ...educe-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskRecoverEvent.java
  85. +108 −21 ...adoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
  86. +2 −7 ...p-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/MapTaskImpl.java
  87. +2 −7 ...apreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ReduceTaskImpl.java
  88. +180 −56 ...preduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
  89. +183 −86 ...doop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
  90. +5 −0 ...e-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
  91. +9 −1 ...ent-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java
  92. +1 −1 ...ce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
  93. +0 −39 ...adoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/Recovery.java
  94. +0 −480 ...apreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
  95. +1 −1 ...apreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
  96. +9 −2 ...hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
  97. +5 −3 ...ce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
  98. +470 −0 ...nt/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
  99. +101 −6 ...oop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
  100. +102 −21 ...p-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
  101. +2 −2 ...pp/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java
  102. +5 −14 ...-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
  103. +1 −3 ...e-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
  104. +4 −1 ...op-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
  105. +8 −3 ...-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
  106. +10 −1 ...src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
  107. +16 −0 ...reduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
  108. +2 −1 ...-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
  109. +0 −14 ...preduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
  110. +12 −1 ...educe-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
  111. +3 −0 ...ce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
  112. +35 −0 ...mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
  113. +5 −0 ...adoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
  114. +1 −1 ...adoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
  115. +42 −13 ...hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestCompletedTask.java
  116. +48 −5 ...p-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
  117. +335 −201 ...op-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
  118. +209 −0 ...oop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryServer.java
  119. +69 −0 ...p-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java
  120. +4 −0 ...es/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist
  121. +5 −0 ...hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
  122. +30 −26 ...e-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
  123. +1 −0 ...reduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
  124. +102 −17 ...ient/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
  125. +10 −4 ...op-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
  126. +6 −0 ...uce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/ExampleDriver.java
  127. +33 −0 hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
  128. +24 −0 hadoop-tools/hadoop-gridmix/pom.xml
  129. +8 −2 hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
  130. +7 −13 hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
  131. +4 −0 hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
  132. +4 −4 hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
  133. +3 −2 ...n/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
  134. +384 −0 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
  135. +2 −2 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
  136. +2 −2 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
  137. +59 −36 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
  138. +430 −0 ...p-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
  139. +989 −0 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java
  140. +202 −0 ...op-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
  141. +5 −6 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
  142. +81 −0 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java
  143. +142 −0 hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java
  144. +414 −0 hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json
  145. +828 −0 hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json
  146. +6 −1 ...-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
  147. +14 −0 hadoop-tools/hadoop-streaming/pom.xml
  148. +3 −4 hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
  149. +18 −0 hadoop-tools/hadoop-streaming/src/test/bin/cat.cmd
  150. +18 −0 hadoop-tools/hadoop-streaming/src/test/bin/xargs_cat.cmd
  151. +5 −1 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
  152. +4 −3 ...op-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
  153. +2 −2 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
  154. +1 −1 ...p-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
  155. +39 −10 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
  156. +1 −1 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
  157. +21 −9 .../hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
  158. +11 −6 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
  159. +3 −3 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
  160. +0 −1 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
  161. +98 −8 hadoop-yarn-project/CHANGES.txt
  162. +21 −0 hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
  163. +1 −7 ...ct/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
  164. +9 −0 ...doop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
  165. +37 −1 ...src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
  166. +8 −16 ...adoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
  167. +0 −24 ...oject/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
  168. +0 −35 ...yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
  169. +39 −20 .../src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
  170. +3 −76 ...rn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
  171. +1 −79 ...arn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
  172. +15 −19 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
  173. +1 −0 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
  174. +2 −5 ...edshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
  175. +2 −1 ...s-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
  176. +5 −4 ...er/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
  177. +5 −3 ...rc/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
  178. +9 −1 ...n-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
  179. +20 −18 ...t/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
  180. +2 −10 ...roject/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
  181. +3 −14 ...-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
  182. +45 −14 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
  183. +7 −5 ...t/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
  184. +9 −7 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
  185. +56 −10 ...ject/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
  186. +1 −1 ...hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
  187. +3 −0 ...common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
  188. +12 −0 ...n/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
  189. +4 −2 .../org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
  190. +1 −1 ...hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeAction.java
  191. +58 −0 ...-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
  192. +1 −1 ...-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
  193. +1 −0 ...doop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
  194. +75 −0 .../test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java
  195. +1 −1 ...r-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
  196. +1 −1 ...-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
  197. +59 −18 ...-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
  198. +1 −1 ...ver-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerEventType.java
  199. +2 −0 ...server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
  200. +89 −23 ...er-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
  201. +37 −0 ...manager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/ResourceLocalizationSpec.java
  202. +118 −0 ...in/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/ResourceLocalizationSpecPBImpl.java
  203. +11 −2 ...rg/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
  204. +5 −10 ...ava/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java
  205. +40 −56 .../hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java
  206. +13 −10 ...rc/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
  207. +3 −0 ...src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
  208. +39 −26 ...main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
  209. +7 −8 ...ain/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
  210. +11 −26 ...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
  211. +149 −0 .../apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java
  212. +4 −0 ...a/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
  213. +155 −11 ...g/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
  214. +30 −5 .../java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
  215. +131 −43 ...apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
  216. +2 −1 ...main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java
  217. +3 −1 ...org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java
  218. +39 −0 ...oop/yarn/server/nodemanager/containermanager/localizer/event/ResourceFailedLocalizationEvent.java
  219. +39 −0 ...manager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerBuilderUtils.java
  220. +1 −1 ...nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
  221. +6 −1 ...server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_service_protos.proto
  222. +4 −3 ...er-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
  223. +8 −2 ...arn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
  224. +11 −25 ...er-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
  225. +116 −38 ...-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
  226. +361 −73 ...er-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
  227. +28 −12 .../java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
  228. +24 −18 ...rc/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
  229. +8 −5 ...test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
  230. +14 −9 ...java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
  231. +14 −12 ...che/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java
  232. +58 −26 .../org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
  233. +112 −0 ...che/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
  234. +268 −7 ...ache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
  235. +73 −21 ...he/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
  236. +3 −1 ...a/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
  237. +7 −6 ...che/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
  238. +8 −7 ...ava/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
  239. +6 −2 ...ver-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
  240. +11 −7 ...r-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
  241. +11 −1 ...manager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
  242. +21 −2 ...-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
  243. +35 −7 ...ver-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
  244. +43 −3 ...-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  245. +19 −10 ...cemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
  246. +20 −21 ...rcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
  247. +6 −4 ...c/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
  248. +10 −5 .../main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
  249. +5 −2 ...ager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
  250. +3 −2 ...sourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
  251. +9 −5 ...r/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
  252. +1 −6 ...ager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
  253. +42 −0 ...java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
  254. +46 −6 ...manager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
  255. +3 −2 ...main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
  256. +2 −2 .../apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
  257. +11 −9 ...ger/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
  258. +11 −6 ...ger/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
  259. +2 −2 ...ger/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
  260. +2 −2 ...anager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
  261. +4 −5 ...rver-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
  262. +3 −4 ...rn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
  263. +2 −1 ...server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
  264. +16 −9 ...rn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
  265. +22 −16 ...rver-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
  266. +5 −0 ...r-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
  267. +1 −1 ...ourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
  268. +1 −1 ...er/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
  269. +54 −5 ...ourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
  270. +5 −2 ...esourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
  271. +80 −5 ...er-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
  272. +34 −1 ...ourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
  273. +24 −1 ...nager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
  274. +1 −1 ...est/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
  275. +8 −5 ...java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
  276. +156 −31 ...ger/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
  277. +36 −0 .../java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
  278. +24 −0 ...src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
  279. +2 −3 ...ger/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
  280. +8 −0 ...src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
  281. +92 −2 ...src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
  282. +3 −2 ...ger/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
  283. +25 −15 ...p-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
  284. +2 −2 ...server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
  285. +23 −7 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
View
@@ -144,10 +144,6 @@
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
@@ -165,6 +165,9 @@ Trunk (Unreleased)
BUG FIXES
+ HADOOP-9451. Fault single-layer config if node group topology is enabled.
+ (Junping Du via llu)
+
HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
@@ -502,6 +505,10 @@ Trunk (Unreleased)
HADOOP-9353. Activate native-win maven profile by default on Windows.
(Arpit Agarwal via szetszwo)
+
+ HADOOP-9437. TestNativeIO#testRenameTo fails on Windows due to assumption
+ that POSIX errno is embedded in NativeIOException. (Chris Nauroth via
+ suresh)
Release 2.0.5-beta - UNRELEASED
@@ -536,6 +543,9 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9358. "Auth failed" log should include exception string (todd)
+ HADOOP-9401. CodecPool: Add counters for number of (de)compressors
+ leased out. (kkambatl via tucu)
+
OPTIMIZATIONS
HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
@@ -604,8 +614,17 @@ Release 2.0.5-beta - UNRELEASED
HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
idle time. (Kai Zheng via atm)
- HADOOP-9357. Fallback to default authority if not specified in FileContext.
- (Andrew Wang via eli)
+ HADOOP-9429. TestConfiguration fails with IBM JAVA. (Amir Sanjar via
+ suresh)
+
+ HADOOP-9222. Cover package with org.apache.hadoop.io.lz4 unit tests (Vadim
+ Bondarev via jlowe)
+
+ HADOOP-9233. Cover package org.apache.hadoop.io.compress.zlib with unit
+ tests (Vadim Bondarev via jlowe)
+
+ HADOOP-9211. Set default max heap size in HADOOP_CLIENT_OPTS to 512m
+ in order to avoid OOME. (Plamen Jeliazkov via shv)
Release 2.0.4-alpha - UNRELEASED
@@ -619,6 +638,9 @@ Release 2.0.4-alpha - UNRELEASED
BUG FIXES
+ HADOOP-9467. Metrics2 record filter should check name as well as tags.
+ (Ganeshan Iyler via llu)
+
HADOOP-9406. hadoop-client leaks dependency on JDK tools jar. (tucu)
HADOOP-9301. hadoop client servlet/jsp/jetty/tomcat JARs creating
@@ -630,6 +652,9 @@ Release 2.0.4-alpha - UNRELEASED
HADOOP-9444. Modify hadoop-policy.xml to replace unexpanded variables to a
default value of '*'. (Roman Shaposhnik via vinodkv)
+ HADOOP-9471. hadoop-client wrongfully excludes jetty-util JAR,
+ breaking webhdfs. (tucu)
+
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
@@ -1593,6 +1618,24 @@ Release 2.0.0-alpha - 05-23-2012
HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via
bobby)
+Release 0.23.8 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ HADOOP-9222. Cover package with org.apache.hadoop.io.lz4 unit tests (Vadim
+ Bondarev via jlowe)
+
+ HADOOP-9233. Cover package org.apache.hadoop.io.compress.zlib with unit
+ tests (Vadim Bondarev via jlowe)
+
Release 0.23.7 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -62,7 +62,7 @@ export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANOD
export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx128m $HADOOP_CLIENT_OPTS"
+export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
# On secure datanodes, user to run the datanode as after dropping privileges
@@ -244,33 +244,17 @@ private FileContext(final AbstractFileSystem defFs,
}
/*
- * Resolve a relative path passed from the user.
- *
- * Relative paths are resolved against the current working directory
- * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
- * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
+ * Remove relative part - return "absolute":
+ * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
+ * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
* ("/foo/bar") are returned unchanged.
*
- * Additionally, we fix malformed URIs that specify a scheme but not an
- * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
- * if it matches the default FS, and let the default FS add in the default
- * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
- *
* Applications that use FileContext should use #makeQualified() since
- * they really want a fully-qualified URI.
+ * they really want a fully qualified URI.
* Hence this method is not called makeAbsolute() and
* has been deliberately declared private.
*/
private Path fixRelativePart(Path p) {
- // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
- if (p.hasSchemeAndNoAuthority()) {
- String scheme = p.toUri().getScheme();
- if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
- p = new Path(p.toUri().getSchemeSpecificPart());
- }
- }
- // Absolute paths are unchanged. Relative paths are resolved against the
- // current working directory.
if (p.isUriPathAbsolute()) {
return p;
} else {
@@ -256,10 +256,6 @@ public boolean isAbsoluteAndSchemeAuthorityNull() {
return (isUriPathAbsolute() &&
uri.getScheme() == null && uri.getAuthority() == null);
}
-
- public boolean hasSchemeAndNoAuthority() {
- return uri.getScheme() != null && uri.getAuthority() == null;
- }
/**
* True if the path component (i.e. directory) of this URI is absolute.
@@ -21,6 +21,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -29,6 +30,10 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
/**
* A global compressor/decompressor pool used to save and reuse
* (possibly native) compression/decompression codecs.
@@ -52,6 +57,29 @@
private static final Map<Class<Decompressor>, List<Decompressor>> decompressorPool =
new HashMap<Class<Decompressor>, List<Decompressor>>();
+ private static <T> LoadingCache<Class<T>, AtomicInteger> createCache(
+ Class<T> klass) {
+ return CacheBuilder.newBuilder().build(
+ new CacheLoader<Class<T>, AtomicInteger>() {
+ @Override
+ public AtomicInteger load(Class<T> key) throws Exception {
+ return new AtomicInteger();
+ }
+ });
+ }
+
+ /**
+ * Map to track the number of leased compressors
+ */
+ private static final LoadingCache<Class<Compressor>, AtomicInteger> compressorCounts =
+ createCache(Compressor.class);
+
+ /**
+ * Map to tracks the number of leased decompressors
+ */
+ private static final LoadingCache<Class<Decompressor>, AtomicInteger> decompressorCounts =
+ createCache(Decompressor.class);
+
private static <T> T borrow(Map<Class<T>, List<T>> pool,
Class<? extends T> codecClass) {
T codec = null;
@@ -90,6 +118,21 @@
}
}
+ @SuppressWarnings("unchecked")
+ private static <T> int getLeaseCount(
+ LoadingCache<Class<T>, AtomicInteger> usageCounts,
+ Class<? extends T> codecClass) {
+ return usageCounts.getUnchecked((Class<T>) codecClass).get();
+ }
+
+ private static <T> void updateLeaseCount(
+ LoadingCache<Class<T>, AtomicInteger> usageCounts, T codec, int delta) {
+ if (codec != null) {
+ Class<T> codecClass = ReflectionUtils.getClass(codec);
+ usageCounts.getUnchecked(codecClass).addAndGet(delta);
+ }
+ }
+
/**
* Get a {@link Compressor} for the given {@link CompressionCodec} from the
* pool or a new one.
@@ -111,6 +154,7 @@ public static Compressor getCompressor(CompressionCodec codec, Configuration con
LOG.debug("Got recycled compressor");
}
}
+ updateLeaseCount(compressorCounts, compressor, 1);
return compressor;
}
@@ -137,6 +181,7 @@ public static Decompressor getDecompressor(CompressionCodec codec) {
LOG.debug("Got recycled decompressor");
}
}
+ updateLeaseCount(decompressorCounts, decompressor, 1);
return decompressor;
}
@@ -155,6 +200,7 @@ public static void returnCompressor(Compressor compressor) {
}
compressor.reset();
payback(compressorPool, compressor);
+ updateLeaseCount(compressorCounts, compressor, -1);
}
/**
@@ -173,5 +219,24 @@ public static void returnDecompressor(Decompressor decompressor) {
}
decompressor.reset();
payback(decompressorPool, decompressor);
+ updateLeaseCount(decompressorCounts, decompressor, -1);
+ }
+
+ /**
+ * Return the number of leased {@link Compressor}s for this
+ * {@link CompressionCodec}
+ */
+ public static int getLeasedCompressorsCount(CompressionCodec codec) {
+ return (codec == null) ? 0 : getLeaseCount(compressorCounts,
+ codec.getCompressorType());
+ }
+
+ /**
+ * Return the number of leased {@link Decompressor}s for this
+ * {@link CompressionCodec}
+ */
+ public static int getLeasedDecompressorsCount(CompressionCodec codec) {
+ return (codec == null) ? 0 : getLeaseCount(decompressorCounts,
+ codec.getDecompressorType());
}
}
@@ -57,7 +57,7 @@
* @return true to accept; false otherwise.
*/
public boolean accepts(MetricsRecord record) {
- return accepts(record.tags());
+ return accepts(record.name()) && accepts(record.tags());
}
}
@@ -191,7 +191,12 @@ public void add(Node node) {
}
rack = getNode(nodeGroup.getNetworkLocation());
- if (rack != null && !(rack instanceof InnerNode)) {
+ // rack should be an innerNode and with parent.
+ // note: rack's null parent case is: node's topology only has one layer,
+ // so rack is recognized as "/" and no parent.
+ // This will be recognized as a node with fault topology.
+ if (rack != null &&
+ (!(rack instanceof InnerNode) || rack.getParent() == null)) {
throw new IllegalArgumentException("Unexpected data node "
+ node.toString()
+ " at an illegal network location");
@@ -27,8 +27,10 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import javax.crypto.SecretKey;
@@ -144,6 +146,10 @@ protected void logUpdateMasterKey(DelegationKey key) throws IOException {
return;
}
+ protected void logExpireToken(TokenIdent ident) throws IOException {
+ return;
+ }
+
/**
* Update the current master key
* This is called once by startThreads before tokenRemoverThread is created,
@@ -363,15 +369,25 @@ public long getRenewDate() {
}
/** Remove expired delegation tokens from cache */
- private synchronized void removeExpiredToken() {
+ private void removeExpiredToken() throws IOException {
long now = Time.now();
- Iterator<DelegationTokenInformation> i = currentTokens.values().iterator();
- while (i.hasNext()) {
- long renewDate = i.next().getRenewDate();
- if (now > renewDate) {
- i.remove();
+ Set<TokenIdent> expiredTokens = new HashSet<TokenIdent>();
+ synchronized (this) {
+ Iterator<Map.Entry<TokenIdent, DelegationTokenInformation>> i =
+ currentTokens.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry<TokenIdent, DelegationTokenInformation> entry = i.next();
+ long renewDate = entry.getValue().getRenewDate();
+ if (renewDate < now) {
+ expiredTokens.add(entry.getKey());
+ i.remove();
+ }
}
}
+ // don't hold lock on 'this' to avoid edit log updates blocking token ops
+ for (TokenIdent ident : expiredTokens) {
+ logExpireToken(ident);
+ }
}
public void stopThreads() {
@@ -816,6 +816,7 @@ JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_nativeio_NativeIO_renameTo0(JNIEnv *env,
jclass clazz, jstring jsrc, jstring jdst)
{
+#ifdef UNIX
const char *src = NULL, *dst = NULL;
src = (*env)->GetStringUTFChars(env, jsrc, NULL);
@@ -829,6 +830,23 @@ jclass clazz, jstring jsrc, jstring jdst)
done:
if (src) (*env)->ReleaseStringUTFChars(env, jsrc, src);
if (dst) (*env)->ReleaseStringUTFChars(env, jdst, dst);
+#endif
+
+#ifdef WINDOWS
+ LPCWSTR src = NULL, dst = NULL;
+
+ src = (LPCWSTR) (*env)->GetStringChars(env, jsrc, NULL);
+ if (!src) goto done; // exception was thrown
+ dst = (LPCWSTR) (*env)->GetStringChars(env, jdst, NULL);
+ if (!dst) goto done; // exception was thrown
+ if (!MoveFile(src, dst)) {
+ throw_ioe(env, GetLastError());
+ }
+
+done:
+ if (src) (*env)->ReleaseStringChars(env, jsrc, src);
+ if (dst) (*env)->ReleaseStringChars(env, jdst, dst);
+#endif
}
/**
Oops, something went wrong.

0 comments on commit 673a7b0

Please sign in to comment.