From bbc16a6e393d3ff3e8859bbe5f3d6b1337aca017 Mon Sep 17 00:00:00 2001 From: AyachiNene <1710082010@qq.com> Date: Tue, 17 Aug 2021 08:37:04 +0800 Subject: [PATCH] [HUDI-2301] fix FileSliceMetrics utils bug --- .../client/utils/FileSliceMetricUtils.java | 15 ++- .../utils/TestFileSliceMetricUtils.java | 108 ++++++++++++++++++ 2 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestFileSliceMetricUtils.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/FileSliceMetricUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/FileSliceMetricUtils.java index 347f86cfba349..09f022bbd76f4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/FileSliceMetricUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/FileSliceMetricUtils.java @@ -44,19 +44,22 @@ public static void addFileSliceCommonMetrics(List fileSlices, Map size >= 0) .reduce(Long::sum).orElse(0L); long baseFileSize = slice.getBaseFile().isPresent() ? slice.getBaseFile().get().getFileSize() : 0L; - // Total read will be the base file + all the log files - totalIORead = FSUtils.getSizeInMB(baseFileSize + totalLogFileSize); + totalIORead += baseFileSize; // Total write will be similar to the size of the base file - totalIOWrite = FSUtils.getSizeInMB(baseFileSize > 0 ? baseFileSize : defaultBaseFileSize); - // Total IO will the the IO for read + write - totalIO = totalIORead + totalIOWrite; + totalIOWrite += baseFileSize > 0 ? baseFileSize : defaultBaseFileSize; } + // Total read will be the base file + all the log files + totalIORead = FSUtils.getSizeInMB(totalIORead + totalLogFileSize); + totalIOWrite = FSUtils.getSizeInMB(totalIOWrite); + + // Total IO will be the IO for read + write + totalIO = totalIORead + totalIOWrite; metrics.put(TOTAL_IO_READ_MB, (double) totalIORead); metrics.put(TOTAL_IO_WRITE_MB, (double) totalIOWrite); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestFileSliceMetricUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestFileSliceMetricUtils.java new file mode 100644 index 0000000000000..1fbd01d576d6a --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestFileSliceMetricUtils.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.client.utils; + +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestFileSliceMetricUtils { + + @Test + public void testFileSliceMetricUtilsWithoutFile() { + Map metrics = new HashMap<>(); + List fileSlices = new ArrayList<>(); + final long defaultBaseFileSize = 10 * 1024 * 1024; + final double epsilon = 1e-5; + FileSliceMetricUtils.addFileSliceCommonMetrics(fileSlices, metrics, defaultBaseFileSize); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_READ_MB), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_WRITE_MB), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_MB), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILE_SIZE), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILES), epsilon); + } + + @Test + public void testFileSliceMetricUtilsWithoutLogFile() { + Map metrics = new HashMap<>(); + List fileSlices = new ArrayList<>(); + final long defaultBaseFileSize = 10 * 1024 * 1024; + final double epsilon = 1e-5; + fileSlices.add(buildFileSlice(15 * 1024 * 1024, new ArrayList<>())); + fileSlices.add(buildFileSlice(20 * 1024 * 1024, new ArrayList<>())); + fileSlices.add(buildFileSlice(0, new ArrayList<>())); + FileSliceMetricUtils.addFileSliceCommonMetrics(fileSlices, metrics, defaultBaseFileSize); + assertEquals(35.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_READ_MB), epsilon); + assertEquals(45.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_WRITE_MB), epsilon); + assertEquals(80.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_MB), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILE_SIZE), epsilon); + assertEquals(0.0, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILES), epsilon); + } + + @Test + public void testFileSliceMetricUtilsWithLogFile() { + Map metrics = new HashMap<>(); + List fileSlices = new ArrayList<>(); + final long defaultBaseFileSize = 10 * 1024 * 1024; + final double epsilon = 1e-5; + fileSlices.add(buildFileSlice(15 * 1024 * 1024, + new ArrayList<>(Arrays.asList(5 * 1024 * 1024L, 3 * 1024 * 1024L)))); + fileSlices.add(buildFileSlice(20 * 1024 * 1024, + new ArrayList<>(Collections.singletonList(2 * 1024 * 1024L)))); + FileSliceMetricUtils.addFileSliceCommonMetrics(fileSlices, metrics, defaultBaseFileSize); + assertEquals(45.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_READ_MB), epsilon); + assertEquals(35.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_WRITE_MB), epsilon); + assertEquals(80.0, metrics.get(FileSliceMetricUtils.TOTAL_IO_MB), epsilon); + assertEquals(10.0 * 1024 * 1024, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILE_SIZE), epsilon); + assertEquals(3.0, metrics.get(FileSliceMetricUtils.TOTAL_LOG_FILES), epsilon); + } + + private FileSlice buildFileSlice(long baseFileLen, List logFileLens) { + final String baseFilePath = ".b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1"; + FileSlice slice = new FileSlice("partition_0", + HoodieActiveTimeline.createNewInstantTime(), + UUID.randomUUID().toString()); + HoodieBaseFile baseFile = new HoodieBaseFile(baseFilePath); + baseFile.setFileLen(baseFileLen); + slice.setBaseFile(baseFile); + int logVersion = 1; + for (long logFileLen : logFileLens) { + String logFilePath = "." + UUID.randomUUID().toString() + "_20170101134598.log." + logVersion; + HoodieLogFile logFile = new HoodieLogFile(logFilePath); + logFile.setFileLen(logFileLen); + slice.addLogFile(logFile); + logVersion++; + } + return slice; + } + +}