From 5c3d1053018dcdf4781a74ae6b8b4ddefe300ddb Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Mon, 1 Dec 2025 20:44:37 +0800 Subject: [PATCH 1/9] add BP-RL BP-DP --- .../org/apache/tsfile/encoding/BDCTest.java | 575 +++++++ .../tsfile/encoding/DPOctadPacking.java | 843 +++++++++ .../encoding/DPOctadPackingSprintz.java | 941 +++++++++++ .../encoding/EfficientOctadPackingMLP.java | 1321 +++++++++++++++ .../EfficientOctadPackingMLPSprintz.java | 1504 +++++++++++++++++ .../tsfile/encoding/FBitpacking512.java | 798 +++++++++ .../apache/tsfile/encoding/FSprintz512.java | 1158 +++++++++++++ .../encoding/RLEPackBitWidthSprintzTest.java | 916 ++++++++++ .../tsfile/encoding/RLEPackBitWidthTest.java | 858 ++++++++++ 9 files changed, 8914 insertions(+) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java new file mode 100644 index 000000000..0db2a6b9e --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java @@ -0,0 +1,575 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.*; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * BDC（dynamic bit packing）实现 - 无 Delta / 无 ZigZag 版本 + * + * 核心函数： + * - encodeDynamicPacking(int[] paddedArray, int packSize, int qmbdLen) -> byte[] + * - decodeDynamicPacking(byte[] encoded, int packSize, int originalLength) -> int[] + * + * 主要改动：bit-packing 部分改用“位缓冲（bit buffer）连续写入 / 读取”的实现， + * 与之前你给出的 pack8Values / unpack8Values 思路一致（按 width 连续写位流）。 + */ +public class BDCTest { + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + private static final int CHUNK_SIZE = 1024; + + // -------------------- 工具 -------------------- + public static int getBitWidth(int num) { + if (num == 0) return 1; + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static void intToBytesBE(int value, byte[] dst, int pos) { + dst[pos] = (byte) (value >> 24); + dst[pos+1] = (byte) (value >> 16); + dst[pos+2] = (byte) (value >> 8); + dst[pos+3] = (byte) (value); + } + public static void packCountToBytesBE(int value, byte[] dst, int pos) { + dst[pos] = (byte) (value); + } + + public static int bytesToIntBE(byte[] src, int pos) { + int v = 0; + v |= (src[pos] & 0xFF) << 24; + v |= (src[pos+1] & 0xFF) << 16; + v |= (src[pos+2] & 0xFF) << 8; + v |= (src[pos+3] & 0xFF); + return v; + } + public static int bytesToPackCount(byte[] src, int pos) { + int v = 0; + v |= (src[pos] & 0xFF); + return v; + } + + // -------------------- 缩放 (无 Delta/ZigZag) -------------------- + private static int[] scaleNumbers(List numbers, int decimalMax) { + BigDecimal scale = BigDecimal.TEN.pow(decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + if (size == 0) return result; + + BigDecimal min = null; + BigDecimal[] scaledValues = new BigDecimal[size]; + + for (int i = 0; i < size; i++) { + BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); + scaledValues[i] = val; + if (min == null || val.compareTo(min) < 0) { + min = val; + } + } + + for (int i = 0; i < size; i++) { + BigDecimal current = scaledValues[i].subtract(min); + result[i] = current.toBigInteger().intValue(); + } + return result; + } + + // -------------------- bit-packing (基于位缓冲，行为与上次 pack8/unpack8 思路一致) -------------------- + /** + * 将 values（任意长度，但通常为 packSize 的整数倍）以 width 位连续写成字节流（高位先出）。 + * 返回字节数组（末尾会按位补齐到整字节）。 + */ + private static byte[] bitPackList(ArrayList values, int width) { + if (values == null || values.isEmpty()) return new byte[0]; + + // 计算需要的字节数 + int totalBits = values.size() * width; + int totalBytes = (totalBits + 7) / 8; + byte[] encoded_result = new byte[totalBytes]; + + // 处理8的倍数个数值 + int encode_pos = bitPacking(values, 0, width, 0, encoded_result); + + // 处理剩余的值（如果不是8的倍数） + int processedCount = (values.size() / 8) * 8; + int remaining = values.size() - processedCount; + + if (remaining > 0) { + // 使用原来的位打包逻辑处理剩余的值 + long bitBuffer = 0L; + int bitCount = 0; + long mask = (width >= 63) ? -1L : ((1L << width) - 1L); + + for (int i = processedCount; i < values.size(); i++) { + int v = values.get(i); + long vv = ((long) v) & mask; + bitBuffer = (bitBuffer << width) | vv; + bitCount += width; + + while (bitCount >= 8) { + int shift = bitCount - 8; + int b = (int) ((bitBuffer >>> shift) & 0xFFL); + encoded_result[encode_pos++] = (byte) b; + bitCount -= 8; + if (bitCount == 0) { + bitBuffer = 0L; + } else { + bitBuffer = bitBuffer & ((1L << bitCount) - 1L); + } + } + } + + // 写剩余的不足一字节的位 + if (bitCount > 0) { + int b = (int) ((bitBuffer << (8 - bitCount)) & 0xFFL); + encoded_result[encode_pos++] = (byte) b; + } + } + + // 创建正确大小的结果数组 + if (encode_pos < totalBytes) { + byte[] result = new byte[encode_pos]; + System.arraycopy(encoded_result, 0, result, 0, encode_pos); + return result; + } + + return encoded_result; + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + // remaining bits for the current unfinished Integer + int leftBit = 0; + + while (valueIdx < 8 + offset) { + // buffer is used for saving 32 bits as a part of result + int buffer = 0; + // remaining size of bits in the 'buffer' + int leftSize = 32; + + // encode the left bits of current Integer to 'buffer' + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + // encode one Integer to the 'buffer' + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + // If the remaining space of the buffer can not save the bits for one Integer, + if (leftSize > 0 && valueIdx < 8 + offset) { + // put the first 'leftSize' bits of the Integer into remaining space of the + // buffer + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + // put the buffer into the final result + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + return encode_pos; + + } + + /** + * 从 encoded[pos..] 连续读取 count 个 width-bit 的值，返回 UnpackResult 包含读出的值与消耗的字节数。 + */ + private static class UnpackResult { + ArrayList values; + int bytesConsumed; + UnpackResult(ArrayList v, int c) { values = v; bytesConsumed = c; } + } + + private static UnpackResult bitUnpackToList(byte[] encoded, int pos, int width, int count) { + ArrayList out = new ArrayList<>(count); + long buffer = 0L; + int totalBits = 0; + int idx = pos; + int consumed = 0; + long mask = (width >= 63) ? -1L : ((1L << width) - 1L); + + while (out.size() < count) { + // 确保 buffer 中至少有 width 位 + while (totalBits < width) { + if (idx >= encoded.length) { + // 输入不足 —— 返回当前已解出的 + return new UnpackResult(out, consumed); + } + buffer = (buffer << 8) | (encoded[idx] & 0xFFL); + idx++; + consumed++; + totalBits += 8; + } + int shift = totalBits - width; + long val = (buffer >>> shift) & mask; + out.add((int) val); + totalBits -= width; + if (totalBits == 0) { + buffer = 0L; + } else { + buffer = buffer & ((1L << totalBits) - 1L); + } + } + return new UnpackResult(out, consumed); + } + + // -------------------- 动态分包（针对每组 group_size 个值的 group-bitwidth 序列） -------------------- + /** + * encodeDynamicPacking: + * 输出格式: + * [0] packCount (1 byte) （原来你写成 header 1 byte） + * for each pack: + * [1 byte] packBitWidth + * [1 byte] numGroups (每组包含 packSize 个值) + * [payload bytes] bit-packed (numGroups * packSize values) using packBitWidth + * + * 说明：为了和你现有调用兼容，header 使用了 1 字节 packCount（如需更大 packCount，请改为 4 字节）。 + */ + public static byte[] encodeDynamicPacking(int[] paddedArray, int packSize, int qmbdLen) { + if (paddedArray == null) return new byte[0]; + int totalGroups = paddedArray.length / packSize; + + ArrayList queueValues = new ArrayList<>(); + int maxBD = 0; + final int SUBHEADER_BITS_ESTIMATE = 40; // 5 bytes header ~= 40 bits + + List packPayloads = new ArrayList<>(); + List packNumGroups = new ArrayList<>(); + List packBitWidths = new ArrayList<>(); + + for (int g = 0; g < totalGroups; g++) { + int start = g * packSize; + int groupMax = 0; + for (int k = 0; k < packSize; k++) { + int val = paddedArray[start + k]; + if (val > groupMax) groupMax = val; + } + int groupBD = getBitWidth(groupMax); + + if (queueValues.isEmpty()) { + for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); + maxBD = groupBD; + if (queueValues.size() / packSize >= qmbdLen) { + int groupCount = queueValues.size() / packSize; + byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); + packPayloads.add(payload); + packNumGroups.add(groupCount); + packBitWidths.add(Math.max(1, maxBD)); + queueValues.clear(); + maxBD = 0; + } + continue; + } + + if (groupBD > maxBD) { + int wastedBits = (groupBD - maxBD) * queueValues.size(); + if (wastedBits >= SUBHEADER_BITS_ESTIMATE || (queueValues.size() / packSize) >= qmbdLen) { + int groupCount = queueValues.size() / packSize; + byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); + packPayloads.add(payload); + packNumGroups.add(groupCount); + packBitWidths.add(Math.max(1, maxBD)); + queueValues.clear(); + maxBD = 0; + + for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); + maxBD = groupBD; + } else { + for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); + maxBD = Math.max(maxBD, groupBD); + if ((queueValues.size() / packSize) >= qmbdLen) { + int groupCount = queueValues.size() / packSize; + byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); + packPayloads.add(payload); + packNumGroups.add(groupCount); + packBitWidths.add(Math.max(1, maxBD)); + queueValues.clear(); + maxBD = 0; + } + } + } else { + for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); + if ((queueValues.size() / packSize) >= qmbdLen) { + int groupCount = queueValues.size() / packSize; + byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); + packPayloads.add(payload); + packNumGroups.add(groupCount); + packBitWidths.add(Math.max(1, maxBD)); + queueValues.clear(); + maxBD = 0; + } + } + } + + if (!queueValues.isEmpty()) { + int groupCount = queueValues.size() / packSize; + byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); + packPayloads.add(payload); + packNumGroups.add(groupCount); + packBitWidths.add(Math.max(1, maxBD)); + queueValues.clear(); + maxBD = 0; + } + +// // 写出最终 byte[] +// ByteArrayOutputStream out = new ByteArrayOutputStream(); +// int packCount = packPayloads.size(); +// // 1 byte packCount +// out.write(packCount & 0xFF); +// try { +// for (int i = 0; i < packCount; i++) { +// int bw = packBitWidths.get(i); +// int ng = packNumGroups.get(i); +// out.write((byte) (bw & 0xFF)); +// out.write((byte) (ng & 0xFF)); +// out.write(packPayloads.get(i)); +// } +// } catch (IOException e) { +// e.printStackTrace(); +// } +// byte[] result = new byte[out.toByteArray().length]; +// for (int i = 0; i < result.length; i++) { +// result[i] = out.toByteArray()[i]; +// } +// return result; + int packCount = packPayloads.size(); + +// 先计算总长度：1 byte packCount + for each pack (1 byte bw + 1 byte numGroups + payload length) + int totalLen = 1; // packCount + for (int i = 0; i < packCount; i++) { + totalLen += 1; // bit width + totalLen += 1; // numGroups + totalLen += packPayloads.get(i).length; // payload + } + +// 分配数组并填充 + byte[] result = new byte[totalLen]; + int pos = 0; + result[pos++] = (byte) (packCount & 0xFF); + + for (int i = 0; i < packCount; i++) { + int bw = packBitWidths.get(i); + int ng = packNumGroups.get(i); + byte[] payload = packPayloads.get(i); + + result[pos++] = (byte) (bw & 0xFF); + result[pos++] = (byte) (ng & 0xFF); +// for (byte b : payload) { +// result[pos++] = (byte) (b & 0xFF); +// } + for (int bi = 0; bi < payload.length; bi++) { + result[pos + bi] = payload[bi]; + } +// System.arraycopy(payload, 0, result, pos, payload.length); + pos += payload.length; + } + +// 返回结果 + return result; + } + + // decode 对应的 encodeDynamicPacking 格式 + public static int[] decodeDynamicPacking(byte[] encoded, int packSize, int originalLength) { + if (encoded == null || encoded.length < 1) return new int[0]; + int pos = 0; + int packCount = bytesToPackCount(encoded, pos); pos += 1; + + ArrayList all = new ArrayList<>(originalLength); + for (int p = 0; p < packCount; p++) { + if (pos >= encoded.length) break; + int bw = encoded[pos++] & 0xFF; + int numGroups = bytesToPackCount(encoded, pos); pos += 1; + int numValues = numGroups * packSize; + UnpackResult ur = bitUnpackToList(encoded, pos, bw, numValues); + all.addAll(ur.values); + pos += ur.bytesConsumed; + } + int[] out = new int[originalLength]; + for (int i = 0; i < originalLength && i < all.size(); i++) out[i] = all.get(i); + if (all.size() < originalLength) { + for (int i = all.size(); i < originalLength; i++) out[i] = 0; + } + return out; + } + + // -------------------- 主测试（参考你原 main） -------------------- + public static void main(String[] args) throws IOException { + System.out.println("\nPerformance Testing (Dynamic pack over 8-values groups)..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BDC"; + File outputDir = new File(outputDirstr); + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + if (!dir.exists()) { + System.err.println("Directory not found: " + directory); + return; + } + + final int groupSize = 8; // 每8个值一个 group + final int qmbdLen = 8; // QMBD 队列上限（可调） + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int time_of_repeat = 50; + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + long startTime = System.nanoTime(); + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + + // pad to multiple of groupSize + int remainder = scaledInts.length % groupSize; + int paddingLength = (remainder == 0) ? 0 : groupSize - remainder; + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); +// int groups = paddedArray.length / groupSize; +// int[] bitWidths = new int[groups]; +// int gidx = 0; +// for (int si = 0; si < paddedArray.length; si += groupSize) { +// long maxInGroup = 0; +// for (int sj = si; sj < si + groupSize; ++sj) { +// long v = paddedArray[sj]; +// if (v > maxInGroup) maxInGroup = v; +// } +// int bitWidth = 0; +// if (maxInGroup > 0) { +// bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); +// } else { +// bitWidth = 0; +// } +// bitWidths[gidx++] = bitWidth; +// } + // encode using dynamic packing over groups of size 8 + byte[] compressed = encodeDynamicPacking(paddedArray, groupSize, qmbdLen); + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += (long) compressed.length * 8L; + + // decode time + long startDec = System.nanoTime(); + int[] decoded = decodeDynamicPacking(compressed, groupSize,paddedArray.length); + long decDur = System.nanoTime() - startDec; + modelDecodeTime += decDur; + +// // 验证（只在第一次迭代验证原始 scaledInts 部分） +// if (j == 0) { +// boolean ok = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decoded[k]) { +// ok = false; +// System.err.println("Mismatch at idx " + k + ": expect " + scaledInts[k] + " got " + decoded[k]); +// break; +// } +// } +// if (ok) System.out.println("Chunk decoded OK."); +// } + } + } + + modelCost = modelCost / time_of_repeat; + modelTime = modelTime / time_of_repeat; + modelDecodeTime = modelDecodeTime / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime == 0 ? 1 : modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime == 0 ? 1 : modelDecodeTime); + + String[] record = { + file.toString(), + "BP_dynamic", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void dummyTest() { + // placeholder for IDE/test runner + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java new file mode 100644 index 000000000..853b0603a --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java @@ -0,0 +1,843 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DPOctadPacking { + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + private static final int CHUNK_SIZE = 1024; + + static String trimStr(String s) { + if (s == null) return ""; + int a = 0; + while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; + if (a == s.length()) return ""; + int b = s.length() - 1; + while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; + return s.substring(a, b + 1); + } + + static String stripEnclosingQuotes(String s) { + if (s == null) return ""; + if (s.length() >= 2) { + char f = s.charAt(0); + char l = s.charAt(s.length() - 1); + if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { + return s.substring(1, s.length() - 1); + } + } + return s; + } + + // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping + static long[] scaleNumbers(List numbers, int decimalMax) { + int n = numbers.size(); + long[] result = new long[n]; + if (n == 0) return result; + + BigDecimal scale = BigDecimal.ONE; + for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); + + BigDecimal[] vals = new BigDecimal[n]; + for (int i = 0; i < n; ++i) { + String s = trimStr(numbers.get(i)); + s = stripEnclosingQuotes(s); + if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } + s = s.replace(",", ""); // remove thousands sep + + // If scientific notation present, BigDecimal can parse it + try { + BigDecimal bd = new BigDecimal(s); + BigDecimal scaled = bd.multiply(scale); + // rounding to nearest whole + BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); + vals[i] = rounded; + } catch (Exception ex) { + // fallback: parse double + try { + double dv = Double.parseDouble(s); + BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); + vals[i] = bd.setScale(0, RoundingMode.HALF_UP); + } catch (Exception ex2) { + System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); + vals[i] = BigDecimal.ZERO; + } + } + } + + // find min + BigDecimal minv = vals[0]; + for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; + + for (int i = 0; i < n; ++i) { + BigDecimal shifted = vals[i].subtract(minv); + // clamp to long range + try { + BigInteger bi = shifted.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (ArithmeticException ae) { + // not an integer exactly: fallback by converting to long with rounding + BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); + try { + BigInteger bi = rounded.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (Exception ex) { + result[i] = 0; + } + } + } + return result; + } + + + // 动态规划结果类 + static class PackingPlan { + int optimalC; + List groupSizes; // 每个分组包含的块数 + List groupBitWidths; // 每个分组的位宽 + int totalCost; + + PackingPlan(int optimalC, List groupSizes, List groupBitWidths, int totalCost) { + this.optimalC = optimalC; + this.groupSizes = groupSizes; + this.groupBitWidths = groupBitWidths; + this.totalCost = totalCost; + } + } + + // 计算最优分组方案的完整实现 + public static PackingPlan computeOptimalPackingPlan(int[] bitWidths, int pack_size) { + int N = bitWidths.length; + if (N == 0) { + return new PackingPlan(0, new ArrayList<>(), new ArrayList<>(), 0); + } + + // 预计算所有区间的最大位宽 + int[][] maxB = new int[N][N]; + for (int l = 0; l < N; l++) { + maxB[l][l] = bitWidths[l]; + for (int r = l + 1; r < N; r++) { + maxB[l][r] = Math.max(maxB[l][r - 1], bitWidths[r]); + } + } + + int minTotalCost = Integer.MAX_VALUE; + int bestC = 1; + List bestGroupSizes = new ArrayList<>(); + List bestGroupBitWidths = new ArrayList<>(); + + int maxPossibleC = 64 - Long.numberOfLeadingZeros(N); + + for (int C = 1; C <= maxPossibleC; C++) { + int low_C = (C == 1) ? 1 : (1 << (C - 1)); + int high_C = Math.min((1 << C) - 1, N); + + // DP表和相关记录数组 + int[][] dp = new int[N + 1][2]; + int[][] prevState = new int[N + 1][2]; + int[][] prevFlag = new int[N + 1][2]; + int[][] packSize = new int[N + 1][2]; + + // 初始化 + for (int i = 0; i <= N; i++) { + dp[i][0] = Integer.MAX_VALUE / 2; + dp[i][1] = Integer.MAX_VALUE / 2; + } + dp[0][0] = 0; + + // 动态规划计算 + for (int i = 1; i <= N; i++) { + for (int k = Math.max(1, i - high_C + 1); k <= i; k++) { + int packLength = i - k + 1; + int currentMaxB = maxB[k - 1][i - 1]; + int packCost = pack_size * packLength * currentMaxB + 6 + C; + + if (packLength < low_C) { + // 小分组，不能改变状态 + if (dp[k - 1][0] + packCost < dp[i][0]) { + dp[i][0] = dp[k - 1][0] + packCost; + prevState[i][0] = k - 1; + prevFlag[i][0] = 0; + packSize[i][0] = packLength; + } + if (dp[k - 1][1] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][1] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 1; + packSize[i][1] = packLength; + } + } else { + // 大分组，可以改变状态到1 + if (dp[k - 1][0] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][0] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 0; + packSize[i][1] = packLength; + } + if (dp[k - 1][1] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][1] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 1; + packSize[i][1] = packLength; + } + } + } + } + + // 回溯构建分组方案 + if (dp[N][1] < minTotalCost) { + minTotalCost = dp[N][1]; + bestC = C; + bestGroupSizes = new ArrayList<>(); + bestGroupBitWidths = new ArrayList<>(); + + // 回溯路径 + int i = N; + int flag = 1; + while (i > 0) { + int size = packSize[i][flag]; + int start = i - size; + int bitWidth = maxB[start][i - 1]; + + bestGroupSizes.add(0, size); + bestGroupBitWidths.add(0, bitWidth); + + i = prevState[i][flag]; + flag = prevFlag[i + size][flag]; // 注意这里要调整索引 + } + } + } + + return new PackingPlan(bestC, bestGroupSizes, bestGroupBitWidths, minTotalCost); + } + + // 计算压缩后数据总大小 + private static int calculateTotalCompressedSize(PackingPlan plan, int pack_size) { + int totalSize = 0; + + // 元数据头大小: C(5bits) + 分组数量(32bits) + totalSize += 6; // C占用5bits + totalSize += 8; // 分组数量占用32bits + + // 每个分组的元数据: packsize(plan.optimalC bits) + 位宽(5bits) + int groupMetadataBits = plan.groupSizes.size() * (plan.optimalC + 6); + totalSize += groupMetadataBits; + + // 压缩数据大小 + for (int i = 0; i < plan.groupSizes.size(); i++) { + int groupBlocks = plan.groupSizes.get(i); + int bitWidth = plan.groupBitWidths.get(i); + int dataBits = groupBlocks * pack_size * bitWidth; + totalSize += dataBits; + } + + // 转换为字节数（向上取整） + return (totalSize + 7) / 8; + } + + // 修改后的bitPacking方法，支持指定的起始位位置 + public static int bitPacking(ArrayList numbers, int start, int bit_width, + int encode_pos, byte[] encoded_result, int startBitPos) { + int block_num = (numbers.size() - start) / 8; + int currentBytePos = encode_pos; + int currentBitPos = startBitPos; + + for (int i = 0; i < block_num; i++) { + // 对每个8值块进行压缩 + for (int j = 0; j < 8; j++) { + int value = numbers.get(start + i * 8 + j); + + // 按位写入 + for (int bit = bit_width - 1; bit >= 0; bit--) { + int currentBit = (value >> bit) & 1; + encoded_result[currentBytePos] |= (currentBit << (7 - currentBitPos)); + currentBitPos++; + + if (currentBitPos == 8) { + currentBytePos++; + currentBitPos = 0; + } + } + } + } + + return currentBytePos; + } + + // 辅助：位写入器 + static class BitWriter { + final byte[] data; + int bytePos = 0; + int bitPos = 0; // 0..7, next bit to write in current byte (7 - bitPos is mask shift) + + BitWriter(byte[] data, int startBytePos) { this.data = data; this.bytePos = startBytePos; } + + // 写 numBits 位（numBits <= 64），value 的低 numBits 位被写出（big-endian bit order） + void writeBits(long value, int numBits) { + for (int i = numBits - 1; i >= 0; --i) { + int bit = (int)((value >> i) & 1L); + data[bytePos] |= (byte)(bit << (7 - bitPos)); + bitPos++; + if (bitPos == 8) { + bitPos = 0; + bytePos++; + } + } + } + + int getBytePos() { return bytePos; } + int getBitPos() { return bitPos; } + // 返回写入结束后占用的字节数（包括部分字节） + int bytesWritten() { + return bytePos + (bitPos > 0 ? 1 : 0); + } + } + + // 辅助：位读取器 + static class BitReader { + final byte[] data; + int bytePos = 0; + int bitPos = 0; + + BitReader(byte[] data, int startBytePos, int startBitPos) { + this.data = data; + this.bytePos = startBytePos; + this.bitPos = startBitPos; + } + + // 读 numBits 位并作为 long 返回（numBits <= 64） + long readBits(int numBits) { + long res = 0L; + for (int i = 0; i < numBits; ++i) { + int bit = (data[bytePos] >> (7 - bitPos)) & 1; + res = (res << 1) | bit; + bitPos++; + if (bitPos == 8) { + bitPos = 0; + bytePos++; + } + } + return res; + } + + int getBytePos() { return bytePos; } + int getBitPos() { return bitPos; } + } + + // ---- 用新的 BitWriter/BitReader 来实现压缩/解压 ---- + public static byte[] compressWithOptimalPacking(long[] paddedArray, int[] bitWidths, int pack_size, int[] encodePos) { + PackingPlan optimalPlan = computeOptimalPackingPlan(bitWidths, pack_size); + int totalSize = calculateTotalCompressedSize(optimalPlan, pack_size); // 以字节计 + byte[] compressedData = new byte[totalSize + 8]; // 预留一点空间以防计算差异（安全余量） + BitWriter writer = new BitWriter(compressedData, 0); + + // 写 C (6 bits) 和 groupCount(8 bits) + writer.writeBits(optimalPlan.optimalC, 6); + writer.writeBits(optimalPlan.groupSizes.size(), 8); + + int dataIndex = 0; + for (int gi = 0; gi < optimalPlan.groupSizes.size(); ++gi) { + int groupBlocks = optimalPlan.groupSizes.get(gi); + int bitWidth = optimalPlan.groupBitWidths.get(gi); + writer.writeBits(groupBlocks, optimalPlan.optimalC); + writer.writeBits(bitWidth, 6); + + int groupDataCount = groupBlocks * pack_size; + for (int i = 0; i < groupDataCount; ++i) { + long v = paddedArray[dataIndex++]; + long mask = (bitWidth == 64) ? ~0L : ((1L << bitWidth) - 1L); + writer.writeBits(v & mask, bitWidth); + } + } + + int finalBytes = writer.bytesWritten(); + if (finalBytes > totalSize) { + // 警告：计算函数可能低估了实际大小。这里扩容并返回实际大小。 + System.err.println(String.format("Warning: calculateTotalCompressedSize underestimated bytes: estimated=%d actual=%d. Returning actual length.", + totalSize, finalBytes)); + byte[] out = new byte[finalBytes]; + System.arraycopy(compressedData, 0, out, 0, finalBytes); + encodePos[0] = finalBytes; + return out; + } else { + // 返回精确长度拷贝 + byte[] out = new byte[finalBytes]; + System.arraycopy(compressedData, 0, out, 0, finalBytes); + encodePos[0] = finalBytes; + return out; + } + } + + public static long[] decompressWithOptimalPacking(byte[] compressedData, int originalLength, int pack_size) { + BitReader reader = new BitReader(compressedData, 0, 0); + int C = (int) reader.readBits(6); + int groupCount = (int) reader.readBits(8); + + // sanity checks + if (C <= 0 || C > 64) throw new IllegalArgumentException("Invalid C read from header: " + C); + if (groupCount < 0 || groupCount > (1 << 20)) throw new IllegalArgumentException("Suspicious groupCount: " + groupCount); + + long[] result = new long[originalLength]; + int resultIndex = 0; + + for (int gi = 0; gi < groupCount; ++gi) { + // 先确保还能读 group header + long remainingBitsBeforeGroup = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); + if (remainingBitsBeforeGroup < C + 6) { + throw new IllegalArgumentException(String.format( + "Not enough bits for group header #%d: need %d but only %d remain (bytePos=%d bitPos=%d totalBytes=%d).", + gi, C + 6, remainingBitsBeforeGroup, reader.getBytePos(), reader.getBitPos(), compressedData.length)); + } + + int groupBlocks = (int) reader.readBits(C); + int bitWidth = (int) reader.readBits(6); + if (bitWidth < 0 || bitWidth > 64) throw new IllegalArgumentException("Invalid bitWidth: " + bitWidth); + + long groupDataCount = (long) groupBlocks * (long) pack_size; + if (groupDataCount < 0 || groupDataCount > (1L << 31)) { + throw new IllegalArgumentException("Suspicious groupDataCount: " + groupDataCount); + } + + long neededDataBits = groupDataCount * (long) bitWidth; + long remainingBitsAfterHeader = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); + if (neededDataBits > remainingBitsAfterHeader) { +// continue; + throw new IllegalArgumentException(String.format( + "Not enough bits for group #%d data: need %d bits but only %d available (bytePos=%d bitPos=%d).", + gi, neededDataBits, remainingBitsAfterHeader, reader.getBytePos(), reader.getBitPos())); + } + + for (long i = 0; i < groupDataCount; ++i) { + long v = reader.readBits(bitWidth); + if (resultIndex < originalLength) result[resultIndex++] = v; + } + } + + return result; + } + + + public static void main(String[] args) throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + // 方法：强化学习 + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + // 1. 找出当前8个元素中的最大值 + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / 8] = bitWidth; + } + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, 8,encodePos); + int cur_cost = res.length*8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + decompressWithOptimalPacking(res, paddedArray.length, 8); + long endDecodeTime = System.nanoTime(); + long decodeDuration = endDecodeTime - startDecodeTime; + + modelDecodeTime += decodeDuration; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /=time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); + double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); + String[] record = { + file.toString(), + "BP-DP", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + } + + } + + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP_vary_pack_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 10; + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + long modelTime = 0; + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, pack_size,encodePos); + int cur_cost = encodePos[0]*8; + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + String[] record = { + file.toString(), + "BP-DP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + writer.close(); + } + + } + + // 新增方法：测试不同chunk size的表现 + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// long[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + long[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + long[] scaledInts_all = new long[totalLength]; + + int currentIndex = 0; + for (long[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); +// +// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + int end = Math.min(i + chunkSize, numbers.size()); + long[] scaledInts = new long[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为pack_size的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking(paddedArray, bitWidths, pack_size, encodePos); + int cur_cost = encodePos[0] * 8; + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "BP-DP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); + } + } +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java new file mode 100644 index 000000000..fc849696c --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java @@ -0,0 +1,941 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +//import org.openjdk.jol.info.ClassLayout; +//import org.openjdk.jol.info.GraphLayout; +import java.math.BigInteger; +public class DPOctadPackingSprintz { + private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv","Basel-wind.csv", + "POI-lon.csv","Air-sensor.csv","Basel-temp.csv"); + + private static final int CHUNK_SIZE = 1024; + + + static String trimStr(String s) { + if (s == null) return ""; + int a = 0; + while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; + if (a == s.length()) return ""; + int b = s.length() - 1; + while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; + return s.substring(a, b + 1); + } + + static String stripEnclosingQuotes(String s) { + if (s == null) return ""; + if (s.length() >= 2) { + char f = s.charAt(0); + char l = s.charAt(s.length() - 1); + if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { + return s.substring(1, s.length() - 1); + } + } + return s; + } + + + static long[] scaleNumbers(List numbers, int decimalMax) { + int n = numbers.size(); + long[] result = new long[n]; + if (n == 0) return result; + + BigDecimal scale = BigDecimal.ONE; + for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); + + BigDecimal[] vals = new BigDecimal[n]; + for (int i = 0; i < n; ++i) { + String s = trimStr(numbers.get(i)); + s = stripEnclosingQuotes(s); + if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } + s = s.replace(",", ""); // remove thousands sep + + // If scientific notation present, BigDecimal can parse it + try { + BigDecimal bd = new BigDecimal(s); + BigDecimal scaled = bd.multiply(scale); + // rounding to nearest whole + BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); + vals[i] = rounded; + } catch (Exception ex) { + // fallback: parse double + try { + double dv = Double.parseDouble(s); + BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); + vals[i] = bd.setScale(0, RoundingMode.HALF_UP); + } catch (Exception ex2) { + System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); + vals[i] = BigDecimal.ZERO; + } + } + } + + // find min + BigDecimal minv = vals[0]; + for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; + + for (int i = 0; i < n; ++i) { + BigDecimal shifted = vals[i].subtract(minv); + // clamp to long range + try { + BigInteger bi = shifted.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (ArithmeticException ae) { + // not an integer exactly: fallback by converting to long with rounding + BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); + try { + BigInteger bi = rounded.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (Exception ex) { + result[i] = 0; + } + } + } + return result; + } + + public static long[] sprintz(long[] numbers) { + int size = numbers.length; + long[] result = new long[size]; + + if (size == 0) return result; // 空数组直接返回 + + long first = numbers[0]; + result[0] = first; + + long prev = first; + for (int i = 1; i < size; i++) { + long current = numbers[i]; + long diff = current - prev; + // ZigZag 编码: 正数 -> 偶数, 负数 -> 奇数 + result[i] = (diff << 1) ^ (diff >> 63); + prev = current; + } + + return result; + } + + // 动态规划结果类 + static class PackingPlan { + int optimalC; + List groupSizes; // 每个分组包含的块数 + List groupBitWidths; // 每个分组的位宽 + int totalCost; + + PackingPlan(int optimalC, List groupSizes, List groupBitWidths, int totalCost) { + this.optimalC = optimalC; + this.groupSizes = groupSizes; + this.groupBitWidths = groupBitWidths; + this.totalCost = totalCost; + } + } + + // 计算最优分组方案的完整实现 + public static PackingPlan computeOptimalPackingPlan(int[] bitWidths, int pack_size) { + int N = bitWidths.length; + if (N == 0) { + return new PackingPlan(0, new ArrayList<>(), new ArrayList<>(), 0); + } + + // 预计算所有区间的最大位宽 + int[][] maxB = new int[N][N]; + for (int l = 0; l < N; l++) { + maxB[l][l] = bitWidths[l]; + for (int r = l + 1; r < N; r++) { + maxB[l][r] = Math.max(maxB[l][r - 1], bitWidths[r]); + } + } + + int minTotalCost = Integer.MAX_VALUE; + int bestC = 1; + List bestGroupSizes = new ArrayList<>(); + List bestGroupBitWidths = new ArrayList<>(); + + int maxPossibleC = 64 - Long.numberOfLeadingZeros(N); + + for (int C = 1; C <= maxPossibleC; C++) { + int low_C = (C == 1) ? 1 : (1 << (C - 1)); + int high_C = Math.min((1 << C) - 1, N); + + // DP表和相关记录数组 + int[][] dp = new int[N + 1][2]; + int[][] prevState = new int[N + 1][2]; + int[][] prevFlag = new int[N + 1][2]; + int[][] packSize = new int[N + 1][2]; + + // 初始化 + for (int i = 0; i <= N; i++) { + dp[i][0] = Integer.MAX_VALUE / 2; + dp[i][1] = Integer.MAX_VALUE / 2; + } + dp[0][0] = 0; + + // 动态规划计算 + for (int i = 1; i <= N; i++) { + for (int k = Math.max(1, i - high_C + 1); k <= i; k++) { + int packLength = i - k + 1; + int currentMaxB = maxB[k - 1][i - 1]; + int packCost = pack_size * packLength * currentMaxB + 6 + C; + + if (packLength < low_C) { + // 小分组，不能改变状态 + if (dp[k - 1][0] + packCost < dp[i][0]) { + dp[i][0] = dp[k - 1][0] + packCost; + prevState[i][0] = k - 1; + prevFlag[i][0] = 0; + packSize[i][0] = packLength; + } + if (dp[k - 1][1] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][1] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 1; + packSize[i][1] = packLength; + } + } else { + // 大分组，可以改变状态到1 + if (dp[k - 1][0] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][0] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 0; + packSize[i][1] = packLength; + } + if (dp[k - 1][1] + packCost < dp[i][1]) { + dp[i][1] = dp[k - 1][1] + packCost; + prevState[i][1] = k - 1; + prevFlag[i][1] = 1; + packSize[i][1] = packLength; + } + } + } + } + + // 回溯构建分组方案 + if (dp[N][1] < minTotalCost) { + minTotalCost = dp[N][1]; + bestC = C; + bestGroupSizes = new ArrayList<>(); + bestGroupBitWidths = new ArrayList<>(); + + // 回溯路径 + int i = N; + int flag = 1; + while (i > 0) { + int size = packSize[i][flag]; + int start = i - size; + int bitWidth = maxB[start][i - 1]; + + bestGroupSizes.add(0, size); + bestGroupBitWidths.add(0, bitWidth); + + i = prevState[i][flag]; + flag = prevFlag[i + size][flag]; // 注意这里要调整索引 + } + } + } + + return new PackingPlan(bestC, bestGroupSizes, bestGroupBitWidths, minTotalCost); + } + + // 计算压缩后数据总大小 + private static int calculateTotalCompressedSize(PackingPlan plan, int pack_size) { + int totalSize = 0; + + // 元数据头大小: C(5bits) + 分组数量(32bits) + totalSize += 6; // C占用5bits + totalSize += 8; // 分组数量占用32bits + + // 每个分组的元数据: packsize(plan.optimalC bits) + 位宽(5bits) + int groupMetadataBits = plan.groupSizes.size() * (plan.optimalC + 6); + totalSize += groupMetadataBits; + + // 压缩数据大小 + for (int i = 0; i < plan.groupSizes.size(); i++) { + int groupBlocks = plan.groupSizes.get(i); + int bitWidth = plan.groupBitWidths.get(i); + int dataBits = groupBlocks * pack_size * bitWidth; + totalSize += dataBits; + } + + // 转换为字节数（向上取整） + return (totalSize + 7) / 8; + } + + + // 修改后的bitPacking方法，支持指定的起始位位置 + public static int bitPacking(ArrayList numbers, int start, int bit_width, + int encode_pos, byte[] encoded_result, int startBitPos) { + int block_num = (numbers.size() - start) / 8; + int currentBytePos = encode_pos; + int currentBitPos = startBitPos; + + for (int i = 0; i < block_num; i++) { + // 对每个8值块进行压缩 + for (int j = 0; j < 8; j++) { + int value = numbers.get(start + i * 8 + j); + + // 按位写入 + for (int bit = bit_width - 1; bit >= 0; bit--) { + int currentBit = (value >> bit) & 1; + encoded_result[currentBytePos] |= (currentBit << (7 - currentBitPos)); + currentBitPos++; + + if (currentBitPos == 8) { + currentBytePos++; + currentBitPos = 0; + } + } + } + } + + return currentBytePos; + } + + // 辅助：位写入器 + private static class BitWriter { + final byte[] data; + int bytePos = 0; + int bitPos = 0; // 0..7, next bit to write in current byte (7 - bitPos is mask shift) + + BitWriter(byte[] data, int startBytePos) { this.data = data; this.bytePos = startBytePos; } + + // 写 numBits 位（numBits <= 64），value 的低 numBits 位被写出（big-endian bit order） + void writeBits(long value, int numBits) { + for (int i = numBits - 1; i >= 0; --i) { + int bit = (int)((value >> i) & 1L); + data[bytePos] |= (byte)(bit << (7 - bitPos)); + bitPos++; + if (bitPos == 8) { + bitPos = 0; + bytePos++; + } + } + } + + int getBytePos() { return bytePos; } + int getBitPos() { return bitPos; } + // 返回写入结束后占用的字节数（包括部分字节） + int bytesWritten() { + return bytePos + (bitPos > 0 ? 1 : 0); + } + } + + static class BitReader { + final byte[] data; + int bytePos = 0; + int bitPos = 0; + + BitReader(byte[] data, int startBytePos, int startBitPos) { + this.data = data; + this.bytePos = startBytePos; + this.bitPos = startBitPos; + } + + // 读 numBits 位并作为 long 返回（numBits <= 64） + long readBits(int numBits) { + long res = 0L; + for (int i = 0; i < numBits; ++i) { + int bit = (data[bytePos] >> (7 - bitPos)) & 1; + res = (res << 1) | bit; + bitPos++; + if (bitPos == 8) { + bitPos = 0; + bytePos++; + } + } + return res; + } + + int getBytePos() { return bytePos; } + int getBitPos() { return bitPos; } + } + + // ---- 用新的 BitWriter/BitReader 来实现压缩/解压 ---- + public static byte[] compressWithOptimalPacking(long[] paddedArray, int[] bitWidths, int pack_size, int[] encodePos) { + PackingPlan optimalPlan = computeOptimalPackingPlan(bitWidths, pack_size); + int totalSize = calculateTotalCompressedSize(optimalPlan, pack_size); // 以字节计 + byte[] compressedData = new byte[totalSize + 8]; // 预留一点空间以防计算差异（安全余量） + BitWriter writer = new BitWriter(compressedData, 0); + + // 写 C (6 bits) 和 groupCount(8 bits) + writer.writeBits(optimalPlan.optimalC, 6); + writer.writeBits(optimalPlan.groupSizes.size(), 8); + + int dataIndex = 0; + for (int gi = 0; gi < optimalPlan.groupSizes.size(); ++gi) { + int groupBlocks = optimalPlan.groupSizes.get(gi); + int bitWidth = optimalPlan.groupBitWidths.get(gi); + writer.writeBits(groupBlocks, optimalPlan.optimalC); + writer.writeBits(bitWidth, 6); + + int groupDataCount = groupBlocks * pack_size; + for (int i = 0; i < groupDataCount; ++i) { + long v = paddedArray[dataIndex++]; + long mask = (bitWidth == 64) ? ~0L : ((1L << bitWidth) - 1L); + writer.writeBits(v & mask, bitWidth); + } + } + + int finalBytes = writer.bytesWritten(); + if (finalBytes > totalSize) { + // 警告：计算函数可能低估了实际大小。这里扩容并返回实际大小。 + System.err.println(String.format("Warning: calculateTotalCompressedSize underestimated bytes: estimated=%d actual=%d. Returning actual length.", + totalSize, finalBytes)); + byte[] out = new byte[finalBytes]; + System.arraycopy(compressedData, 0, out, 0, finalBytes); + encodePos[0] = finalBytes; + return out; + } else { + // 返回精确长度拷贝 + byte[] out = new byte[finalBytes]; + System.arraycopy(compressedData, 0, out, 0, finalBytes); + encodePos[0] = finalBytes; + return out; + } + } + + // 解压缩方法 + public static long[] decompressWithOptimalPacking(byte[] compressedData, int originalLength, int pack_size) { + BitReader reader = new BitReader(compressedData, 0, 0); + int C = (int) reader.readBits(6); + int groupCount = (int) reader.readBits(8); + + // sanity checks + if (C <= 0 || C > 64) throw new IllegalArgumentException("Invalid C read from header: " + C); + if (groupCount < 0 || groupCount > (1 << 20)) throw new IllegalArgumentException("Suspicious groupCount: " + groupCount); + + long[] result = new long[originalLength]; + int resultIndex = 0; + + for (int gi = 0; gi < groupCount; ++gi) { + // 先确保还能读 group header + long remainingBitsBeforeGroup = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); + if (remainingBitsBeforeGroup < C + 6) { + throw new IllegalArgumentException(String.format( + "Not enough bits for group header #%d: need %d but only %d remain (bytePos=%d bitPos=%d totalBytes=%d).", + gi, C + 6, remainingBitsBeforeGroup, reader.getBytePos(), reader.getBitPos(), compressedData.length)); + } + + int groupBlocks = (int) reader.readBits(C); + int bitWidth = (int) reader.readBits(6); + if (bitWidth < 0 || bitWidth > 64) throw new IllegalArgumentException("Invalid bitWidth: " + bitWidth); + + long groupDataCount = (long) groupBlocks * (long) pack_size; + if (groupDataCount < 0 || groupDataCount > (1L << 31)) { + throw new IllegalArgumentException("Suspicious groupDataCount: " + groupDataCount); + } + + long neededDataBits = groupDataCount * (long) bitWidth; + long remainingBitsAfterHeader = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); + if (neededDataBits > remainingBitsAfterHeader) { +// continue; + throw new IllegalArgumentException(String.format( + "Not enough bits for group #%d data: need %d bits but only %d available (bytePos=%d bitPos=%d).", + gi, neededDataBits, remainingBitsAfterHeader, reader.getBytePos(), reader.getBitPos())); + } + + for (long i = 0; i < groupDataCount; ++i) { + long v = reader.readBits(bitWidth); + if (resultIndex < originalLength) result[resultIndex++] = v; + } + } + + return result; + } + + + // 读取指定位数的辅助函数 + private static int readBits(byte[] data, int bytePos, int bitPos, int numBits) { + int result = 0; + for (int i = 0; i < numBits; i++) { + int bit = (data[bytePos] >> (7 - bitPos)) & 1; + result = (result << 1) | bit; + bitPos++; + if (bitPos == 8) { + bytePos++; + bitPos = 0; + } + } + return result; + } + + public static long[] sprintzDecode(long[] encodedData) { + int size = encodedData.length; + long[] result = new long[size]; + + if (size == 0) return result; + + // 第一个元素是原始值 + result[0] = encodedData[0]; + + // 后续元素需要ZigZag解码和累加 + long prev = result[0]; + for (int i = 1; i < size; i++) { + long zigzagEncoded = encodedData[i]; + long diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 + result[i] = prev + diff; + prev = result[i]; + } + + return result; + } + + public static void main(String[] args) throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); +// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP"; + File outputDir = new File(outputDirstr); + +// RLDecisionModel trainedModel = trainModel(20, csvFilePath); + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; +// if(!file.getName().equals("Stocks-DE.csv")) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 30; +// System.out.println(numbers.size()); + + + // 方法：强化学习 +// long modelStart = System.nanoTime(); + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + long[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + long[] scaledInts = sprintz(scalingInt); + + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + // 1. 找出当前8个元素中的最大值 + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / 8] = bitWidth; + } + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, 8,encodePos); + int cur_cost = encodePos[0]*8; +// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 + long duration = System.nanoTime() - startTime; + + + long startDecodeTime = System.nanoTime(); + long[] sprintz_encode_data = decompressWithOptimalPacking(res, paddedArray.length, 8); + long[] decode_data = sprintzDecode(sprintz_encode_data); + long endDecodeTime = System.nanoTime(); + long decodeDuration = endDecodeTime - startDecodeTime; + + modelDecodeTime += decodeDuration; + modelTime += (duration); + modelCost += cur_cost; +// if(i==0) +// for (int episode = 0; episode < 10; episode++) { +// trainEpisode(scaledInts, episode); +// } +// List optimalK = predictOptimalK(scaledInts); +// System.out.println("Optimal k sequence: " + optimalK); + } + + } + modelCost /=time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); + double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); + String[] record = { + file.toString(), + "Sprintz-DP", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); +// break; + } + + } + + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); +// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP_vary_pack_size"; + File outputDir = new File(outputDirstr); + + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; +// if(!file.getName().equals("Stocks-DE.csv")) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + System.out.println(pack_size); + int modelCost = 0; + long modelTime = 0; + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + + long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + long[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, pack_size,encodePos); + int cur_cost = encodePos[0]*8; + +// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; +// if(i==0) +// for (int episode = 0; episode < 10; episode++) { +// trainEpisode(scaledInts, episode); +// } +// List optimalK = predictOptimalK(scaledInts); +// System.out.println("Optimal k sequence: " + optimalK); + } + + } + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + String[] record = { + file.toString(), + "SPRINTZ-DP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + writer.close(); +// break; + } + + } + + // 新增方法：测试不同chunk size的表现 + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// long[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + long[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + long[] scaledInts_all = new long[totalLength]; + + int currentIndex = 0; + for (long[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); +// +// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + int end = Math.min(i + chunkSize, numbers.size()); + long[] scaledInt = new long[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + + long[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为pack_size的倍数 + long[] paddedArray = new long[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + long maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + int[] encodePos = new int[1]; + byte[] res = compressWithOptimalPacking(paddedArray, bitWidths, pack_size, encodePos); + int cur_cost = encodePos[0] * 8; + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "Sprintz-DP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); + } + } + +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java new file mode 100644 index 000000000..90eb6e422 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java @@ -0,0 +1,1321 @@ +package org.apache.iotdb.tsfile.encoding; +// EfficientOctadPackingMLP_optimized.java +// Java 17+ + +import org.junit.Test; + +import java.io.*; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.*; + +public class EfficientOctadPackingMLP { + + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + static final int CHUNK_SIZE = 1024; + static final int INPUT_DIM = 5; + static final int HIDDEN_DIM = 48; + + // ========== CSV loader & scaling helpers ========== + static List> loadDataFromCSV(String filename) { + List> sequences = new ArrayList<>(); + Pattern pattern = Pattern.compile("\"?\\[([0-9,\\s]+)\\]\"?"); + try (BufferedReader br = new BufferedReader(new FileReader(filename))) { + String line; + boolean firstLine = true; + while ((line = br.readLine()) != null) { + if (firstLine) { firstLine = false; continue; } + Matcher m = pattern.matcher(line); + if (m.find()) { + String data = m.group(1); + List arr = new ArrayList<>(); + String[] tokens = data.split(","); + for (String t : tokens) { + String s = t.trim(); + if (s.isEmpty()) continue; + try { + arr.add(Integer.parseInt(s)); + } catch (Exception ex) { /* ignore */ } + } + if (!arr.isEmpty()) sequences.add(arr); + } + } + } catch (IOException e) { + System.err.println("Error opening CSV: " + filename); + } + return sequences; + } + + static String trimStr(String s) { + if (s == null) return ""; + int a = 0; + while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; + if (a == s.length()) return ""; + int b = s.length() - 1; + while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; + return s.substring(a, b + 1); + } + + static String stripEnclosingQuotes(String s) { + if (s == null) return ""; + if (s.length() >= 2) { + char f = s.charAt(0); + char l = s.charAt(s.length() - 1); + if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { + return s.substring(1, s.length() - 1); + } + } + return s; + } + + // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping + static long[] scaleNumbers(List numbers, int decimalMax) { + int n = numbers.size(); + long[] result = new long[n]; + if (n == 0) return result; + + BigDecimal scale = BigDecimal.ONE; + for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); + + BigDecimal[] vals = new BigDecimal[n]; + for (int i = 0; i < n; ++i) { + String s = trimStr(numbers.get(i)); + s = stripEnclosingQuotes(s); + if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } + s = s.replace(",", ""); // remove thousands sep + + try { + BigDecimal bd = new BigDecimal(s); + BigDecimal scaled = bd.multiply(scale); + BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); + vals[i] = rounded; + } catch (Exception ex) { + try { + double dv = Double.parseDouble(s); + BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); + vals[i] = bd.setScale(0, RoundingMode.HALF_UP); + } catch (Exception ex2) { + System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); + vals[i] = BigDecimal.ZERO; + } + } + } + + BigDecimal minv = vals[0]; + for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; + + for (int i = 0; i < n; ++i) { + BigDecimal shifted = vals[i].subtract(minv); + try { + BigInteger bi = shifted.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (ArithmeticException ae) { + BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); + try { + BigInteger bi = rounded.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (Exception ex) { + result[i] = 0; + } + } + } + return result; + } + // ========== Pack / Result / DecisionPoint ========== + static class Pack { + int size = 0; + int maxBitWidth = 0; + int startIndex = 0; + List indices = new ArrayList<>(); + List bitWidths = new ArrayList<>(); + + void addOctad(int index, int bitWidth) { + if (size == 0) { + startIndex = index; + maxBitWidth = bitWidth; + } else { + if (bitWidth > maxBitWidth) maxBitWidth = bitWidth; + } + indices.add(index); + bitWidths.add(bitWidth); + size++; + } + + long dataCost(long pack_size) { + return pack_size * size * (long) maxBitWidth; + } + + int logSize() { + if (size <= 0) return 0; + return 32 - Integer.numberOfLeadingZeros(size); + } + } + + static class PackingResult { + int packCount = 0; + long dataCostA = 0; + int bitWidthCostB = 0; + int packSizeCostC = 0; + long totalCost = 0; + List packs = new ArrayList<>(); + byte[] compressedData; + + void calculateCost(int maxLog) { + bitWidthCostB = 6 * packCount; + packSizeCostC = packCount * maxLog; + totalCost = dataCostA + bitWidthCostB + packSizeCostC; + } + + @Override + public String toString() { + return String.format("Packs: %d, Cost: %d (A=%d, B=%d, C=%d)", packCount, totalCost, dataCostA, bitWidthCostB, packSizeCostC); + } + } + + private static double safeLog(double p) { + return Math.log(Math.max(p, 1e-8)); + } + + static class DecisionPoint { + int currentPackSize; + int currentPackMaxB; + int newOctadB; + int packCount; + int currentMaxLog; + boolean action; + float probability; + + DecisionPoint(int cps, int cpm, int nob, int pc, int cml, boolean a, float p) { + currentPackSize = cps; + currentPackMaxB = cpm; + newOctadB = nob; + packCount = pc; + currentMaxLog = cml; + action = a; + probability = p; + } + } + + // ========== 2-layer MLP policy with REINFORCE ========== + static class RLDecisionModel { + float[] W1; // size HIDDEN_DIM * INPUT_DIM + float[] b1; // size HIDDEN_DIM + float[] W2; // size HIDDEN_DIM + float b2; + + float explorationRate = 0.3f; + float learningRate = 0.01f; + + Random rng; + + RLDecisionModel() { + rng = new Random(); + W1 = new float[HIDDEN_DIM * INPUT_DIM]; + b1 = new float[HIDDEN_DIM]; + W2 = new float[HIDDEN_DIM]; + for (int i = 0; i < W1.length; ++i) W1[i] = randUniform(-0.08f, 0.08f); + for (int i = 0; i < b1.length; ++i) b1[i] = randUniform(-0.08f, 0.08f); + for (int i = 0; i < W2.length; ++i) W2[i] = randUniform(-0.08f, 0.08f); + b2 = randUniform(-0.08f, 0.08f); + } + + private float randUniform(float a, float b) { + return a + rng.nextFloat() * (b - a); + } + + static float relu(float x) { return x > 0.0f ? x : 0.0f; } + static float reluDeriv(float x) { return x > 0.0f ? 1.0f : 0.0f; } + static float sigmoid(float x) { + if (x >= 0) { + double z = Math.exp(-x); + return (float)(1.0 / (1.0 + z)); + } else { + double z = Math.exp(x); + return (float)(z / (1.0 + z)); + } + } + + float forwardProb(float[] feat, float[] outHidden, float[] outZ1) { + if (outHidden != null) Arrays.fill(outHidden, 0.0f); + if (outZ1 != null) Arrays.fill(outZ1, 0.0f); + + for (int h = 0; h < HIDDEN_DIM; ++h) { + float z = b1[h]; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) { + z += W1[base + j] * feat[j]; + } + if (outZ1 != null) outZ1[h] = z; + float hval = relu(z); + if (outHidden != null) outHidden[h] = hval; + } + + float z2 = b2; + if (outHidden != null) { + for (int h = 0; h < HIDDEN_DIM; ++h) z2 += W2[h] * outHidden[h]; + } else { + for (int h = 0; h < HIDDEN_DIM; ++h) { + float z = b1[h]; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) z += W1[base + j] * feat[j]; + float hval = relu(z); + z2 += W2[h] * hval; + } + } + return sigmoid(z2); + } + + float forwardProb(float[] feat) { + return forwardProb(feat, null, null); + } + + float train(List decisions, float reward) { + if (decisions == null || decisions.isEmpty()) return 0.0f; + + explorationRate *= 0.99f; + if (explorationRate < 0.05f) explorationRate = 0.05f; + + float[] dW1 = new float[W1.length]; + float[] db1 = new float[b1.length]; + float[] dW2 = new float[W2.length]; + float db2 = 0.0f; + + float totalLoss = 0.0f; + + float[] feat = new float[INPUT_DIM]; + float[] hidden = new float[HIDDEN_DIM]; + float[] z1 = new float[HIDDEN_DIM]; + + for (DecisionPoint dp : decisions) { + feat[0] = dp.currentPackSize / 100.0f; + feat[1] = dp.currentPackMaxB / 64.0f; + feat[2] = dp.newOctadB / 64.0f; + feat[3] = dp.packCount / 100.0f; + feat[4] = dp.currentMaxLog / 10.0f; + + float p = forwardProb(feat, hidden, z1); + + float pClipped = Math.min(Math.max(p, 1e-6f), 1.0f - 1e-6f); + + float piA = dp.action ? pClipped : (1.0f - pClipped); + if (piA <= 0.0f) { + piA = 1e-6f; + } + float lossI = -reward * (float) Math.log(piA); + + if (Float.isNaN(lossI) || Float.isInfinite(lossI)) { + System.err.printf("Warning: loss is NaN or Infinite. p=%.8f, piA=%.8f, reward=%.8f\n", p, piA, reward); + lossI = 0.0f; + } + + totalLoss += lossI; + + float dL_dz2 = reward * (p - (dp.action ? 1.0f : 0.0f)); + + for (int h = 0; h < HIDDEN_DIM; ++h) { + dW2[h] += dL_dz2 * hidden[h]; + } + db2 += dL_dz2; + + for (int h = 0; h < HIDDEN_DIM; ++h) { + float w2h = W2[h]; + float dh = dL_dz2 * w2h; + float dReLU = reluDeriv(z1[h]); + float dZ1 = dh * dReLU; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) { + dW1[base + j] += dZ1 * feat[j]; + } + db1[h] += dZ1; + } + } + + float lr = learningRate; + for (int i = 0; i < W1.length; ++i) { + W1[i] -= lr * dW1[i]; + W1[i] = clip(W1[i], -10f, 10f); + } + for (int i = 0; i < b1.length; ++i) { + b1[i] -= lr * db1[i]; + b1[i] = clip(b1[i], -10f, 10f); + } + for (int i = 0; i < W2.length; ++i) { + W2[i] -= lr * dW2[i]; + W2[i] = clip(W2[i], -10f, 10f); + } + b2 -= lr * db2; + b2 = clip(b2, -10f, 10f); + + return totalLoss; + } + + static float clip(float v, float low, float high) { + return Math.min(Math.max(v, low), high); + } + + } + + // ========== Bitpacking utility methods (legacy 8-values helpers kept) ========== + public static int getBitWidth(int num) { + if (num == 0) + return 1; + else + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + + while (valueIdx < 8 + offset) { + int buffer = 0; + int leftSize = 32; + + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + if (leftSize > 0 && valueIdx < 8 + offset) { + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + return encode_pos; + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + private static class BitWriter { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private long acc = 0L; // holds currently buffered bits (lowest "accBits" bits are valid) + private int accBits = 0; // number of bits in acc + + // writeBits expects bits in LSB-aligned form (i.e., "masked" value). We append MSB-first as: acc = (acc << bitCount) | bits + void writeBits(long bits, int bitCount) { + if (bitCount == 0) return; + if (bitCount == 64) { + // split into two 32-bit writes to avoid shifting by 64 + writeBits((bits >>> 32) & 0xFFFFFFFFL, 32); + writeBits(bits & 0xFFFFFFFFL, 32); + return; + } + long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); + long v = bits & mask; + acc = (acc << bitCount) | v; + accBits += bitCount; + while (accBits >= 8) { + int shift = accBits - 8; + int outb = (int) ((acc >>> shift) & 0xFFL); + out.write(outb); + if (shift > 0) { + acc &= ((1L << shift) - 1L); + } else { + acc = 0L; + } + accBits = shift; + } + } + + byte[] finish() { + if (accBits > 0) { + int outb = (int) ((acc << (8 - accBits)) & 0xFFL); + out.write(outb); + acc = 0L; + accBits = 0; + } + return out.toByteArray(); + } + } +// private static class BitReader { +// final byte[] data; +// private long acc = 0L; +// private int accBits = 0; +// private int idx = 0; +// +// BitReader(byte[] data) { +// this.data = data; +// } +// +// long readBits(int bitCount) throws IOException { +// if (bitCount == 0) return 0L; +// while (accBits < bitCount) { +// if (idx < data.length) { +// acc = (acc << 8) | (data[idx++] & 0xFFL); +// accBits += 8; +// } else { +// // pad with zeros if stream ends prematurely +// acc = (acc << (bitCount - accBits)); +// accBits = bitCount; +// } +// } +// int shift = accBits - bitCount; +// long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); +// long v = (acc >>> shift) & mask; +// if (shift > 0) { +// acc &= ((1L << shift) - 1L); +// } else { +// acc = 0L; +// } +// accBits = shift; +// return v; +// } +// } +public static final class BitReader { + private final byte[] data; + private int bitPos; // global bit position from start of data[] + + public BitReader(byte[] data) { + this(data, 0); + } + + public BitReader(byte[] data, int byteOffset) { + this.data = data; + this.bitPos = byteOffset * 8; + } + + /** + * Read n bits (0 <= n <= 64), return as unsigned long. + */ + public long readBits(int n) { + if (n == 0) return 0L; + if (n < 0 || n > 64) { + throw new IllegalArgumentException("n must be between 0 and 64"); + } + + long result = 0L; + int bitsRemaining = n; + + while (bitsRemaining > 0) { + int byteIndex = bitPos >>> 3; // current byte + int bitOffset = bitPos & 7; // offset inside byte [0..7] + + // 添加边界检查 + if (byteIndex >= data.length) { + // 如果已经超出数据范围，填充0并返回 + result = (result << bitsRemaining); + bitPos += bitsRemaining; + return result; + } + + int bitsFromCurrentByte = Math.min(8 - bitOffset, bitsRemaining); + + // Load byte as unsigned + int curByte = data[byteIndex] & 0xFF; + + // Shift to get the relevant bits + int shift = 8 - bitOffset - bitsFromCurrentByte; + int chunk = (curByte >>> shift) & ((1 << bitsFromCurrentByte) - 1); + + result = (result << bitsFromCurrentByte) | chunk; + + bitPos += bitsFromCurrentByte; + bitsRemaining -= bitsFromCurrentByte; + } + + return result; + } + + /** + * @return total bits consumed since creation / since byteOffset + */ + public int consumedBits() { + return bitPos; + } + + /** + * @return current bit position (alias) + */ + public int bitPosition() { + return bitPos; + } + + /** + * @return remaining bits available for reading + */ + public int remainingBits() { + return (data.length * 8) - bitPos; + } +} + private static byte[] performBitPackingCompression64_fast(long[] dataArray, List packs, int pack_size, int originalLength) throws IOException { + // 计算一些元信息 + int totalPacks = packs.size(); + int maxOctadsInAnyPack = 0; + for (Pack p : packs) if (p.size > maxOctadsInAnyPack) maxOctadsInAnyPack = p.size; + + // bits needed to encode counts in range [0..maxOctadsInAnyPack] + int bitsForCount = 1; + while ((1L << bitsForCount) <= maxOctadsInAnyPack) bitsForCount++; + if (bitsForCount <= 0) bitsForCount = 1; + + // 准备输出缓冲（先写 header 的整数字段，meta/data 用 BitWriter 位流） + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + + dos.writeByte(totalPacks); + dos.writeByte(bitsForCount); // 1 byte is enough to carry this small number + dos.flush(); + + // --- Meta bitstream --- + BitWriter metaWriter = new BitWriter(); + + // For each pack: write pack.size using bitsForCount bits, then for each octad write its bitWidth using 6 bits + for (Pack pack : packs) { + // write octad count + metaWriter.writeBits(pack.size, bitsForCount); + metaWriter.writeBits(pack.bitWidths.get(0), 6); +// // write each octad's bitWidth using 6 bits. +// // NOTE: we map bitWidth==64 -> store 63 (as sentinel). 解码端须按此约定把 63 映射回 64。 +// for (int i = 0; i < pack.size; ++i) { +// int bw = pack.bitWidths.get(i); +// int store = bw; +// if (bw == 64) store = 63; +// if (store < 0) store = 0; +// if (store > 63) store = 63; // safety clamp +// metaWriter.writeBits(store, 5); +// } + } + + byte[] metaBytes = metaWriter.finish(); + dos.write(metaBytes); + dos.flush(); + + // --- Data bitstream --- + BitWriter dataWriter = new BitWriter(); + + // For each pack, find packMaxBitWidth and write each group's pack_size values using packMaxBitWidth bits + for (Pack pack : packs) { + int packMaxBW = pack.maxBitWidth; + // no change for packMaxBW == 64: BitWriter supports splitting 64 into two 32-bit writes + for (int i = 0; i < pack.size; ++i) { + int originalGroupIndex = pack.indices.get(0); + int startPos = originalGroupIndex * pack_size; + for (int j = 0; j < pack_size; ++j) { + long val; +// if (startPos + j < dataArray.length) { + val = dataArray[startPos + j]; +// } else { +// val = 0L; +// } + // mask value to packMaxBW bits (if packMaxBW == 64, mask preserves full 64 bits) + long mask; + if (packMaxBW == 0) { + dataWriter.writeBits(0L, 0); // nothing to write + } else { + if (packMaxBW == 64) { + // write full 64-bit value (BitWriter handles split) + dataWriter.writeBits(val, 64); + } else { + mask = (1L << packMaxBW) - 1L; + long masked = val & mask; + dataWriter.writeBits(masked, packMaxBW); + } + } + } + } + } + + byte[] dataBytes = dataWriter.finish(); + dos.write(dataBytes); + dos.flush(); + + return baos.toByteArray(); + } + + // ========== packOctads (updated to accept originalLength for compression) ========== + static PackingResult packOctads(List bitWidths, RLDecisionModel model, List decisionTrace, int pack_size, long[] dataArray, int originalLength) { + PackingResult result = new PackingResult(); + Pack currentPack = new Pack(); + int globalMaxLog = 0; + int packCount = 0; + + Random localRng = ThreadLocalRandom.current(); + + for (int i = 0; i < bitWidths.size(); ++i) { + int b = bitWidths.get(i); + + if (currentPack.size == 0) { + currentPack.addOctad(i, b); + } else if (b == currentPack.maxBitWidth) { + currentPack.addOctad(i, b); + } else { + float[] feat = new float[INPUT_DIM]; + feat[0] = currentPack.size / 100.0f; + feat[1] = currentPack.maxBitWidth / 64.0f; + feat[2] = b / 64.0f; + feat[3] = packCount / 100.0f; + feat[4] = globalMaxLog / 10.0f; + + float probability = model.forwardProb(feat); + + boolean shouldMerge; + if (localRng.nextFloat() < model.explorationRate) { + shouldMerge = (localRng.nextFloat() > 0.5f); + } else { + shouldMerge = probability > 0.5f; + } + + if (decisionTrace != null) { + decisionTrace.add(new DecisionPoint(currentPack.size, currentPack.maxBitWidth, b, packCount, globalMaxLog, shouldMerge, probability)); + } + + if (shouldMerge) { + currentPack.addOctad(i, b); + } else { + result.dataCostA += currentPack.dataCost(pack_size); + int logSize = currentPack.logSize(); + if (logSize > globalMaxLog) globalMaxLog = logSize; + result.packs.add(currentPack); + packCount++; + + currentPack = new Pack(); + currentPack.addOctad(i, b); + } + } + } + + if (currentPack.size > 0) { + result.dataCostA += currentPack.dataCost(pack_size); + int logSize = currentPack.logSize(); + if (logSize > globalMaxLog) globalMaxLog = logSize; + result.packs.add(currentPack); + packCount++; + } + + result.packCount = packCount; + result.calculateCost(globalMaxLog); + + // 执行实际的bitpacking压缩（如果提供了 dataArray） + if (dataArray != null) { + try { + result.compressedData = performBitPackingCompression64_fast(dataArray, result.packs, pack_size, originalLength); + } catch (IOException e) { + System.err.println("Compression failed: " + e.getMessage()); + result.compressedData = null; + } + } + + return result; + } + +// public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { +// try { +// // 这里需要根据实际的压缩格式来解析 +// // 假设compressedData包含bit-packed数据 +// ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); +// DataInputStream dis = new DataInputStream(bais); +// +// // 读取bit-packed数据 +// int totalGroups = bitWidths.length; +// long[] result = new long[totalGroups * packSize]; +// int resultIndex = 0; +// +// BitReader reader = new BitReader(compressedData); +// +// for (int g = 0; g < totalGroups; ++g) { +// int bw = bitWidths[g]; +// for (int k = 0; k < packSize; ++k) { +// if (bw == 0) { +// result[resultIndex++] = 0L; +// } else if (bw == 64) { +// long high = reader.readBits(32); +// long low = reader.readBits(32); +// long v = (high << 32) | (low & 0xFFFFFFFFL); +// result[resultIndex++] = v; +// } else { +// long v = reader.readBits(bw); +// result[resultIndex++] = v; +// } +// } +// } +// +// // 只取原始长度的数据并Sprintz解码 +// return Arrays.copyOf(result, originalLength); +// +// } catch (IOException e) { +// System.err.println("Fast decompression failed: " + e.getMessage()); +// return new long[0]; +// } +// } + + public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { + if (compressedData == null || compressedData.length == 0) { + System.err.println("Compressed data is null or empty"); + return new long[0]; + } + + try { + ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); + DataInputStream dis = new DataInputStream(bais); + + // === Header === + int totalPacks = dis.readUnsignedByte(); + int bitsForCount = dis.readUnsignedByte(); + + // === Read meta bitstream === + int metaStartOffset = 2; // two bytes read + BitReader metaReader = new BitReader(compressedData, metaStartOffset); + + // 解析每个pack的信息 + List packInfos = new ArrayList<>(); + int totalValuesToDecode = 0; + + for (int p = 0; p < totalPacks; ++p) { + int octadCount = (int) metaReader.readBits(bitsForCount); + int packBitWidth = (int) metaReader.readBits(6); + if (packBitWidth == 63) packBitWidth = 64; + + packInfos.add(new PackInfo(octadCount, packBitWidth)); + totalValuesToDecode += octadCount * packSize; + } + + // 计算数据部分的起始位置 + int metaBitsUsed = metaReader.consumedBits(); + int dataStartByte = metaStartOffset + (metaBitsUsed + 7) / 8; + + // 检查数据起始位置是否超出压缩数据范围 + if (dataStartByte >= compressedData.length) { +// System.err.println("Data start position exceeds compressed data length"); + return new long[0]; + } + + // === Data bitstream === + BitReader dataReader = new BitReader(compressedData, dataStartByte); + List resultList = new ArrayList<>(); + + // === 按pack解码数据 === + for (PackInfo packInfo : packInfos) { + int octadCount = packInfo.octadCount; + int bitWidth = packInfo.bitWidth; + + // 检查剩余数据是否足够 + if (dataReader.remainingBits() < (long) octadCount * packSize * bitWidth) { +// System.err.println("Insufficient data for decoding pack. Expected: " + +// (octadCount * packSize * bitWidth) + " bits, Available: " + +// dataReader.remainingBits() + " bits"); + break; + } + + // 每个octad包含packSize个值 + for (int i = 0; i < octadCount; ++i) { + for (int j = 0; j < packSize; ++j) { + long value; + if (bitWidth == 0) { + value = 0L; + } else if (bitWidth == 64) { + // 64位特殊处理：分成两个32位读取 + long high = dataReader.readBits(32); + long low = dataReader.readBits(32); + value = (high << 32) | low; + } else { + value = dataReader.readBits(bitWidth); + } + resultList.add(value); + } + } + } + + // 转换为数组并截取到原始长度 + long[] result = new long[Math.min(resultList.size(), originalLength)]; + for (int i = 0; i < result.length; i++) { + result[i] = resultList.get(i); + } + +// System.out.println("Decompression completed: " + result.length + " values decoded"); + return result; + + } catch (Exception e) { + System.err.println("Fast decompression failed: " + e.getMessage()); + e.printStackTrace(); + return new long[0]; + } + } + + // 辅助类，存储pack信息 + static class PackInfo { + int octadCount; + int bitWidth; + + PackInfo(int octadCount, int bitWidth) { + this.octadCount = octadCount; + this.bitWidth = bitWidth; + } + } + // ========== Training loop (trainModel) ========== + static RLDecisionModel trainModel(int epochs, String csvFilePath) { + System.err.println("Training RL model from CSV data..."); + RLDecisionModel model = new RLDecisionModel(); + List> sequences = loadDataFromCSV(csvFilePath); + if (sequences.isEmpty()) { + System.err.println("No data loaded from CSV. Returning initial model."); + return model; + } + System.err.println("Loaded " + sequences.size() + " sequences from CSV"); + + List decisionTrace = new ArrayList<>(); + for (int epoch = 1; epoch <= epochs; ++epoch) { + long startTime = System.nanoTime(); + float totalReward = 0.0f; + float totalLoss = 0.0f; + int processedSequences = 0; + + for (List bitWidths : sequences) { + decisionTrace.clear(); + // training does not perform actual compression, pass dataArray=null and originalLength=0 + PackingResult result = packOctads(bitWidths, model, decisionTrace, 8, null, 0); + + float reward = (float) result.totalCost / 500000.0f; + totalReward += reward; + + float loss = model.train(decisionTrace, reward); + totalLoss += loss; + processedSequences++; + } + + long durationMs = (System.nanoTime() - startTime) / 1_000_000L; + + if (epoch % 10 == 0 || epoch == 1 || epoch == epochs) { + System.out.printf("Epoch %d: Avg Reward = %.6f, Avg Loss = %.6f, Time = %d ms%n", + epoch, + totalReward / processedSequences, + totalLoss / processedSequences, + durationMs); + } else { + System.out.printf("Epoch %d done. Time = %d ms%n", epoch, durationMs); + } + } + + return model; + } + + + // ========== performanceTest (更新版本，包含解压测试) ========== + static void performanceTest(RLDecisionModel model, String directory, String outputDirStr) { + System.out.println("\nPerformance Testing..."); + Path outdir = Paths.get(outputDirStr); + try { + if (!Files.exists(outdir)) Files.createDirectories(outdir); + } catch (IOException e) { + System.err.println("Cannot create output dir: " + outputDirStr); + return; + } + + try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { + for (Path entry : ds) { + if (!Files.isRegularFile(entry)) continue; + String fname = entry.getFileName().toString(); + if (IGNORE_FILES.contains(fname)) continue; + + System.out.println("Processing " + fname + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + + try (BufferedReader br = Files.newBufferedReader(entry)) { + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(","); + for (String token : tokens) { + String t = trimStr(token); + if (!t.isEmpty()) { + numbers.add(t); + int dec = 0; + int pos = t.indexOf('.'); + if (pos != -1) dec = t.length() - pos - 1; + decimalPlaces.add(dec); + } + } + } + } catch (IOException e) { + System.err.println("Cannot open " + entry.toString()); + continue; + } + + if (numbers.isEmpty()) continue; + + Path outPath = outdir.resolve(fname); + try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { + // 更新表头，增加解压吞吐率列 + writer.write("Input Direction,Encoding Algorithm,Encoding Time,Decoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); + + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; // 新增：解压时间统计 + long compressedSize = 0; + + for (int rep = 0; rep < time_of_repeat; ++rep) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(numbers.size(), i + CHUNK_SIZE); + if (end - i <= 2) continue; + List chunkNumbers = numbers.subList(i, end); + int decimalMax = 0; + for (int k = i; k < end; ++k) { + if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); + } + + long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + + int remainder = scaledInts.length % pack_size; + int padding = (remainder == 0) ? 0 : pack_size - remainder; + long[] padded = new long[scaledInts.length + padding]; + System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); + if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); + + int groups = padded.length / pack_size; + int[] bitWidths = new int[groups]; + int gidx = 0; + for (int si = 0; si < padded.length; si += pack_size) { + long maxInGroup = 0; + for (int sj = si; sj < si + pack_size; ++sj) { + long v = padded[sj]; + if (v > maxInGroup) maxInGroup = v; + } + int bitWidth = 0; + if (maxInGroup > 0) { + bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + } else { + bitWidth = 0; + } + bitWidths[gidx++] = bitWidth; + } + + // pass original length (un-padded) so decoder can trim + List bitWidthsList = new ArrayList<>(groups); + for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); + + PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += (res.compressedData.length*8); + + // 新增：测试解压性能 + if (res.compressedData != null) { + long startDecodeTime = System.nanoTime(); + long[] decompressed = fastDecompress(res.compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + } + +// if (rep == 0) { +// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; +// } + } + } + + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + modelDecodeTime /= time_of_repeat; // 平均解压时间 + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) modelTime; // points/s + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) modelDecodeTime; // points/s + + writer.write(entry + ","); + writer.write("BP-RL,"); + writer.write(modelTime_throughput + ","); + writer.write(modelDecodeTime_throughput + ","); // 解压吞吐率 + writer.write(numbers.size() + ","); + writer.write(modelCost + ","); + writer.write(pack_size + ","); + writer.write(model_ratio + "\n"); + +// System.out.println("Pack Size: " + pack_size); +// System.out.println("Encoding throughput: " + modelTime_throughput + " points/s"); +// System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " points/s"); +// System.out.println("Compression ratio: " + model_ratio); + } + } catch (IOException e) { + System.err.println("Error writing output file for " + fname); + } + } + } catch (IOException e) { + System.err.println("Error iterating directory: " + directory); + } + } + + public static void main(String[] args) { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL"; + + int epochs = 20; + + if (args.length >= 1) trainCsv = args[0]; + if (args.length >= 2) dataDir = args[1]; + if (args.length >= 3) outDir = args[2]; + + RLDecisionModel model = new RLDecisionModel(); + if (!trainCsv.isEmpty()) { + model = trainModel(epochs, trainCsv); + } else { + System.err.println("No training CSV given. Using randomly initialized RL model."); + } + + if (!dataDir.isEmpty()) { + performanceTest(model, dataDir, outDir); + } else { + System.err.println("No data directory provided for performanceTest. Exiting."); + } + } + @Test + public void TestVarPackSize() { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL_vary_pack_size"; + + int epochs = 80; + + RLDecisionModel model = new RLDecisionModel(); + model = trainModel(epochs, trainCsv); + performanceTest(model, dataDir, outDir); + } + // ========== performanceTest with variable chunk sizes ========== + static void performanceTestVariableChunkSize(RLDecisionModel model, String directory, String outputDirStr) { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + Path outdir = Paths.get(outputDirStr); + try { + if (!Files.exists(outdir)) Files.createDirectories(outdir); + } catch (IOException e) { + System.err.println("Cannot create output dir: " + outputDirStr); + return; + } + + // Define the chunk sizes to test (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { + for (Path entry : ds) { + if (!Files.isRegularFile(entry)) continue; + String fname = entry.getFileName().toString(); + if (IGNORE_FILES.contains(fname)) continue; + + System.out.println("Processing " + fname + " with variable chunk sizes..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + + try (BufferedReader br = Files.newBufferedReader(entry)) { + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(","); + for (String token : tokens) { + String t = trimStr(token); + if (!t.isEmpty()) { + numbers.add(t); + int dec = 0; + int pos = t.indexOf('.'); + if (pos != -1) dec = t.length() - pos - 1; + decimalPlaces.add(dec); + } + } + } + } catch (IOException e) { + System.err.println("Cannot open " + entry.toString()); + continue; + } + + if (numbers.isEmpty()) continue; + + Path outPath = outdir.resolve(fname.replace(".", "_chunksize_test.")); + try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { + writer.write("m,Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); + + int time_of_repeat = 50; // Reduced for faster testing with multiple chunk sizes + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + long[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + long[] scaledInts_all = new long[totalLength]; + + int currentIndex = 0; + for (long[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // Test each chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for (int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + long modelCost = 0; + long modelTime = 0; + long compressedSize = 0; + + for (int rep = 0; rep < time_of_repeat; ++rep) { + for (int i = 0; i < numbers.size(); i += chunkSize) { +// int end = Math.min(numbers.size(), i + chunkSize); +// if (end - i <= 2) continue; +// List chunkNumbers = numbers.subList(i, end); +// int decimalMax = 0; +// for (int k = i; k < end; ++k) { +// if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); +// } +// +// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + int end = Math.min(i + chunkSize, numbers.size()); + long[] scaledInts = new long[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + int remainder = scaledInts.length % pack_size; + int padding = (remainder == 0) ? 0 : pack_size - remainder; + long[] padded = new long[scaledInts.length + padding]; + System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); + if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); + + int groups = padded.length / pack_size; + int[] bitWidths = new int[groups]; + int gidx = 0; + for (int si = 0; si < padded.length; si += pack_size) { + long maxInGroup = 0; + for (int sj = si; sj < si + pack_size; ++sj) { + long v = padded[sj]; + if (v > maxInGroup) maxInGroup = v; + } + int bitWidth = 0; + if (maxInGroup > 0) { + bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + } else { + bitWidth = 0; + } + bitWidths[gidx++] = bitWidth; + } + + // pass original length (un-padded) so decoder can trim + List bitWidthsList = new ArrayList<>(groups); + for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); + + PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += (res.compressedData.length* 8L); + +// if (rep == 0) { +// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; +// } + } + } + + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms + + writer.write(String.valueOf(chunkSize/8) + ","); + writer.write(entry.toString() + ","); + writer.write("BP-RL,"); + writer.write(String.valueOf(modelTime_throughput) + ","); + writer.write(String.valueOf(numbers.size()) + ","); + writer.write(String.valueOf(modelCost) + ","); + writer.write(String.valueOf(pack_size) + ","); + writer.write(String.valueOf(model_ratio) + "\n"); + } + } + } catch (IOException e) { + System.err.println("Error writing output file for " + fname); + } + } + } catch (IOException e) { + System.err.println("Error iterating directory: " + directory); + } + } + + @Test + public void TestVariableChunkSize() { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL_vary_m"; + + int epochs = 20; + + RLDecisionModel model = new RLDecisionModel(); + model = trainModel(epochs, trainCsv); + performanceTestVariableChunkSize(model, dataDir, outDir); + } +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java new file mode 100644 index 000000000..b0a0f8c4b --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java @@ -0,0 +1,1504 @@ +package org.apache.iotdb.tsfile.encoding; +// EfficientOctadPackingMLP_optimized.java +// Java 17+ + +import org.junit.Test; + +import java.io.*; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.*; + +public class EfficientOctadPackingMLPSprintz { + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + static final int CHUNK_SIZE = 1024; + static final int INPUT_DIM = 5; + static final int HIDDEN_DIM = 48; + + // ========== Pack / Result / DecisionPoint ========== + static class Pack { + int size = 0; + int maxBitWidth = 0; + int startIndex = 0; + List indices = new ArrayList<>(); + List bitWidths = new ArrayList<>(); + + void addOctad(int index, int bitWidth) { + if (size == 0) { + startIndex = index; + maxBitWidth = bitWidth; + } else { + if (bitWidth > maxBitWidth) maxBitWidth = bitWidth; + } + indices.add(index); + bitWidths.add(bitWidth); + size++; + } + + long dataCost(long pack_size) { + return pack_size * size * (long) maxBitWidth; + } + + int logSize() { + if (size <= 0) return 0; + return 32 - Integer.numberOfLeadingZeros(size); + } + } + + static class PackingResult { + int packCount = 0; + long dataCostA = 0; + int bitWidthCostB = 0; + int packSizeCostC = 0; + long totalCost = 0; + List packs = new ArrayList<>(); + byte[] compressedData; + + void calculateCost(int maxLog) { + bitWidthCostB = 6 * packCount; + packSizeCostC = packCount * maxLog; + totalCost = dataCostA + bitWidthCostB + packSizeCostC; + } + + @Override + public String toString() { + return String.format("Packs: %d, Cost: %d (A=%d, B=%d, C=%d)", packCount, totalCost, dataCostA, bitWidthCostB, packSizeCostC); + } + } + + private static double safeLog(double p) { + return Math.log(Math.max(p, 1e-8)); + } + + static class DecisionPoint { + int currentPackSize; + int currentPackMaxB; + int newOctadB; + int packCount; + int currentMaxLog; + boolean action; + float probability; + + DecisionPoint(int cps, int cpm, int nob, int pc, int cml, boolean a, float p) { + currentPackSize = cps; + currentPackMaxB = cpm; + newOctadB = nob; + packCount = pc; + currentMaxLog = cml; + action = a; + probability = p; + } + } + + // ========== 2-layer MLP policy with REINFORCE ========== + static class RLDecisionModel { + float[] W1; // size HIDDEN_DIM * INPUT_DIM + float[] b1; // size HIDDEN_DIM + float[] W2; // size HIDDEN_DIM + float b2; + + float explorationRate = 0.3f; + float learningRate = 0.01f; + + Random rng; + + RLDecisionModel() { + rng = new Random(); + W1 = new float[HIDDEN_DIM * INPUT_DIM]; + b1 = new float[HIDDEN_DIM]; + W2 = new float[HIDDEN_DIM]; + for (int i = 0; i < W1.length; ++i) W1[i] = randUniform(-0.08f, 0.08f); + for (int i = 0; i < b1.length; ++i) b1[i] = randUniform(-0.08f, 0.08f); + for (int i = 0; i < W2.length; ++i) W2[i] = randUniform(-0.08f, 0.08f); + b2 = randUniform(-0.08f, 0.08f); + } + + private float randUniform(float a, float b) { + return a + rng.nextFloat() * (b - a); + } + + static float relu(float x) { return x > 0.0f ? x : 0.0f; } + static float reluDeriv(float x) { return x > 0.0f ? 1.0f : 0.0f; } + static float sigmoid(float x) { + if (x >= 0) { + double z = Math.exp(-x); + return (float)(1.0 / (1.0 + z)); + } else { + double z = Math.exp(x); + return (float)(z / (1.0 + z)); + } + } + + float forwardProb(float[] feat, float[] outHidden, float[] outZ1) { + if (outHidden != null) Arrays.fill(outHidden, 0.0f); + if (outZ1 != null) Arrays.fill(outZ1, 0.0f); + + for (int h = 0; h < HIDDEN_DIM; ++h) { + float z = b1[h]; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) { + z += W1[base + j] * feat[j]; + } + if (outZ1 != null) outZ1[h] = z; + float hval = relu(z); + if (outHidden != null) outHidden[h] = hval; + } + + float z2 = b2; + if (outHidden != null) { + for (int h = 0; h < HIDDEN_DIM; ++h) z2 += W2[h] * outHidden[h]; + } else { + for (int h = 0; h < HIDDEN_DIM; ++h) { + float z = b1[h]; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) z += W1[base + j] * feat[j]; + float hval = relu(z); + z2 += W2[h] * hval; + } + } + return sigmoid(z2); + } + + float forwardProb(float[] feat) { + return forwardProb(feat, null, null); + } + + float train(List decisions, float reward) { + if (decisions == null || decisions.isEmpty()) return 0.0f; + + explorationRate *= 0.99f; + if (explorationRate < 0.05f) explorationRate = 0.05f; + + float[] dW1 = new float[W1.length]; + float[] db1 = new float[b1.length]; + float[] dW2 = new float[W2.length]; + float db2 = 0.0f; + + float totalLoss = 0.0f; + + float[] feat = new float[INPUT_DIM]; + float[] hidden = new float[HIDDEN_DIM]; + float[] z1 = new float[HIDDEN_DIM]; + + for (DecisionPoint dp : decisions) { + feat[0] = dp.currentPackSize / 100.0f; + feat[1] = dp.currentPackMaxB / 64.0f; + feat[2] = dp.newOctadB / 64.0f; + feat[3] = dp.packCount / 100.0f; + feat[4] = dp.currentMaxLog / 10.0f; + + float p = forwardProb(feat, hidden, z1); + + float pClipped = Math.min(Math.max(p, 1e-6f), 1.0f - 1e-6f); + + float piA = dp.action ? pClipped : (1.0f - pClipped); + if (piA <= 0.0f) { + piA = 1e-6f; + } + float lossI = -reward * (float) Math.log(piA); + + if (Float.isNaN(lossI) || Float.isInfinite(lossI)) { + System.err.printf("Warning: loss is NaN or Infinite. p=%.8f, piA=%.8f, reward=%.8f\n", p, piA, reward); + lossI = 0.0f; + } + + totalLoss += lossI; + + float dL_dz2 = reward * (p - (dp.action ? 1.0f : 0.0f)); + + for (int h = 0; h < HIDDEN_DIM; ++h) { + dW2[h] += dL_dz2 * hidden[h]; + } + db2 += dL_dz2; + + for (int h = 0; h < HIDDEN_DIM; ++h) { + float w2h = W2[h]; + float dh = dL_dz2 * w2h; + float dReLU = reluDeriv(z1[h]); + float dZ1 = dh * dReLU; + int base = h * INPUT_DIM; + for (int j = 0; j < INPUT_DIM; ++j) { + dW1[base + j] += dZ1 * feat[j]; + } + db1[h] += dZ1; + } + } + + float lr = learningRate; + for (int i = 0; i < W1.length; ++i) { + W1[i] -= lr * dW1[i]; + W1[i] = clip(W1[i], -10f, 10f); + } + for (int i = 0; i < b1.length; ++i) { + b1[i] -= lr * db1[i]; + b1[i] = clip(b1[i], -10f, 10f); + } + for (int i = 0; i < W2.length; ++i) { + W2[i] -= lr * dW2[i]; + W2[i] = clip(W2[i], -10f, 10f); + } + b2 -= lr * db2; + b2 = clip(b2, -10f, 10f); + + return totalLoss; + } + + static float clip(float v, float low, float high) { + return Math.min(Math.max(v, low), high); + } + + } + + // ========== Bitpacking utility methods (legacy 8-values helpers kept) ========== + public static int getBitWidth(int num) { + if (num == 0) + return 1; + else + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + + while (valueIdx < 8 + offset) { + int buffer = 0; + int leftSize = 32; + + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + if (leftSize > 0 && valueIdx < 8 + offset) { + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + return encode_pos; + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + // ========== 64-bit-capable canonical encoder/decoder (fast, primitive-based) ========== + // DecodedResult wraps decoded padded values and originalLength (so caller can trim). + public static class DecodedResult { + public final long[] values; // padded values: totalGroups * packSize + public final int originalLength; // original (unpadded) length + public final int packSize; + public DecodedResult(long[] values, int originalLength, int packSize) { + this.values = values; + this.originalLength = originalLength; + this.packSize = packSize; + } + } + + + + + // ========== CSV loader & scaling helpers ========== + static List> loadDataFromCSV(String filename) { + List> sequences = new ArrayList<>(); + Pattern pattern = Pattern.compile("\"?\\[([0-9,\\s]+)\\]\"?"); + try (BufferedReader br = new BufferedReader(new FileReader(filename))) { + String line; + boolean firstLine = true; + while ((line = br.readLine()) != null) { + if (firstLine) { firstLine = false; continue; } + Matcher m = pattern.matcher(line); + if (m.find()) { + String data = m.group(1); + List arr = new ArrayList<>(); + String[] tokens = data.split(","); + for (String t : tokens) { + String s = t.trim(); + if (s.isEmpty()) continue; + try { + arr.add(Integer.parseInt(s)); + } catch (Exception ex) { /* ignore */ } + } + if (!arr.isEmpty()) sequences.add(arr); + } + } + } catch (IOException e) { + System.err.println("Error opening CSV: " + filename); + } + return sequences; + } + + static String trimStr(String s) { + if (s == null) return ""; + int a = 0; + while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; + if (a == s.length()) return ""; + int b = s.length() - 1; + while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; + return s.substring(a, b + 1); + } + + static String stripEnclosingQuotes(String s) { + if (s == null) return ""; + if (s.length() >= 2) { + char f = s.charAt(0); + char l = s.charAt(s.length() - 1); + if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { + return s.substring(1, s.length() - 1); + } + } + return s; + } + + // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping + static long[] scaleNumbers(List numbers, int decimalMax) { + int n = numbers.size(); + long[] result = new long[n]; + if (n == 0) return result; + + BigDecimal scale = BigDecimal.ONE; + for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); + + BigDecimal[] vals = new BigDecimal[n]; + for (int i = 0; i < n; ++i) { + String s = trimStr(numbers.get(i)); + s = stripEnclosingQuotes(s); + if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } + s = s.replace(",", ""); // remove thousands sep + + try { + BigDecimal bd = new BigDecimal(s); + BigDecimal scaled = bd.multiply(scale); + BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); + vals[i] = rounded; + } catch (Exception ex) { + try { + double dv = Double.parseDouble(s); + BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); + vals[i] = bd.setScale(0, RoundingMode.HALF_UP); + } catch (Exception ex2) { + System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); + vals[i] = BigDecimal.ZERO; + } + } + } + + BigDecimal minv = vals[0]; + for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; + + for (int i = 0; i < n; ++i) { + BigDecimal shifted = vals[i].subtract(minv); + try { + BigInteger bi = shifted.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (ArithmeticException ae) { + BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); + try { + BigInteger bi = rounded.toBigIntegerExact(); + if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; + else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; + else result[i] = bi.longValue(); + } catch (Exception ex) { + result[i] = 0; + } + } + } + return result; + } + public static long[] sprintz(long[] numbers) { + int size = numbers.length; + long[] result = new long[size]; + + if (size == 0) return result; // 空数组直接返回 + + long first = numbers[0]; + result[0] = first; + + long prev = first; + for (int i = 1; i < size; i++) { + long current = numbers[i]; + long diff = current - prev; + // ZigZag 编码: 正数 -> 偶数, 负数 -> 奇数 + result[i] = (diff << 1) ^ (diff >> 63); + prev = current; + } + + return result; + } + /** + * Sprintz解码 - 从差分编码恢复原始数据 (long版本) + */ + public static long[] sprintzDecode(long[] encodedData) { + int size = encodedData.length; + long[] result = new long[size]; + + if (size == 0) return result; + + // 第一个元素是原始值 + result[0] = encodedData[0]; + + // 后续元素需要ZigZag解码和累加 + long prev = result[0]; + for (int i = 1; i < size; i++) { + long zigzagEncoded = encodedData[i]; + // ZigZag解码: (n >>> 1) ^ (-(n & 1)) + long diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); + result[i] = prev + diff; + prev = result[i]; + } + + return result; + } + private static class BitWriter { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private long acc = 0L; // holds currently buffered bits (lowest "accBits" bits are valid) + private int accBits = 0; // number of bits in acc + + // writeBits expects bits in LSB-aligned form (i.e., "masked" value). We append MSB-first as: acc = (acc << bitCount) | bits + void writeBits(long bits, int bitCount) { + if (bitCount == 0) return; + if (bitCount == 64) { + // split into two 32-bit writes to avoid shifting by 64 + writeBits((bits >>> 32) & 0xFFFFFFFFL, 32); + writeBits(bits & 0xFFFFFFFFL, 32); + return; + } + long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); + long v = bits & mask; + acc = (acc << bitCount) | v; + accBits += bitCount; + while (accBits >= 8) { + int shift = accBits - 8; + int outb = (int) ((acc >>> shift) & 0xFFL); + out.write(outb); + if (shift > 0) { + acc &= ((1L << shift) - 1L); + } else { + acc = 0L; + } + accBits = shift; + } + } + + byte[] finish() { + if (accBits > 0) { + int outb = (int) ((acc << (8 - accBits)) & 0xFFL); + out.write(outb); + acc = 0L; + accBits = 0; + } + return out.toByteArray(); + } + } + // private static class BitReader { +// final byte[] data; +// private long acc = 0L; +// private int accBits = 0; +// private int idx = 0; +// +// BitReader(byte[] data) { +// this.data = data; +// } +// +// long readBits(int bitCount) throws IOException { +// if (bitCount == 0) return 0L; +// while (accBits < bitCount) { +// if (idx < data.length) { +// acc = (acc << 8) | (data[idx++] & 0xFFL); +// accBits += 8; +// } else { +// // pad with zeros if stream ends prematurely +// acc = (acc << (bitCount - accBits)); +// accBits = bitCount; +// } +// } +// int shift = accBits - bitCount; +// long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); +// long v = (acc >>> shift) & mask; +// if (shift > 0) { +// acc &= ((1L << shift) - 1L); +// } else { +// acc = 0L; +// } +// accBits = shift; +// return v; +// } +// } + public static final class BitReader { + private final byte[] data; + private int bitPos; // global bit position from start of data[] + + public BitReader(byte[] data) { + this(data, 0); + } + + public BitReader(byte[] data, int byteOffset) { + this.data = data; + this.bitPos = byteOffset * 8; + } + + /** + * Read n bits (0 <= n <= 64), return as unsigned long. + */ + public long readBits(int n) { + if (n == 0) return 0L; + if (n < 0 || n > 64) { + throw new IllegalArgumentException("n must be between 0 and 64"); + } + + long result = 0L; + int bitsRemaining = n; + + while (bitsRemaining > 0) { + int byteIndex = bitPos >>> 3; // current byte + int bitOffset = bitPos & 7; // offset inside byte [0..7] + + // 添加边界检查 + if (byteIndex >= data.length) { + // 如果已经超出数据范围，填充0并返回 + result = (result << bitsRemaining); + bitPos += bitsRemaining; + return result; + } + + int bitsFromCurrentByte = Math.min(8 - bitOffset, bitsRemaining); + + // Load byte as unsigned + int curByte = data[byteIndex] & 0xFF; + + // Shift to get the relevant bits + int shift = 8 - bitOffset - bitsFromCurrentByte; + int chunk = (curByte >>> shift) & ((1 << bitsFromCurrentByte) - 1); + + result = (result << bitsFromCurrentByte) | chunk; + + bitPos += bitsFromCurrentByte; + bitsRemaining -= bitsFromCurrentByte; + } + + return result; + } + + /** + * @return total bits consumed since creation / since byteOffset + */ + public int consumedBits() { + return bitPos; + } + + /** + * @return current bit position (alias) + */ + public int bitPosition() { + return bitPos; + } + + /** + * @return remaining bits available for reading + */ + public int remainingBits() { + return (data.length * 8) - bitPos; + } + } + private static byte[] performBitPackingCompression64_fast(long[] dataArray, List packs, int pack_size, int originalLength) throws IOException { + // 计算一些元信息 + int totalPacks = packs.size(); + int maxOctadsInAnyPack = 0; + for ( Pack p : packs) if (p.size > maxOctadsInAnyPack) maxOctadsInAnyPack = p.size; + + // bits needed to encode counts in range [0..maxOctadsInAnyPack] + int bitsForCount = 1; + while ((1L << bitsForCount) <= maxOctadsInAnyPack) bitsForCount++; + if (bitsForCount <= 0) bitsForCount = 1; + + // 准备输出缓冲（先写 header 的整数字段，meta/data 用 BitWriter 位流） + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + + dos.writeByte(totalPacks); + dos.writeByte(bitsForCount); // 1 byte is enough to carry this small number + dos.flush(); + + // --- Meta bitstream --- + BitWriter metaWriter = new BitWriter(); + + // For each pack: write pack.size using bitsForCount bits, then for each octad write its bitWidth using 6 bits + for ( Pack pack : packs) { + // write octad count + metaWriter.writeBits(pack.size, bitsForCount); + metaWriter.writeBits(pack.bitWidths.get(0), 6); +// // write each octad's bitWidth using 6 bits. +// // NOTE: we map bitWidth==64 -> store 63 (as sentinel). 解码端须按此约定把 63 映射回 64。 +// for (int i = 0; i < pack.size; ++i) { +// int bw = pack.bitWidths.get(i); +// int store = bw; +// if (bw == 64) store = 63; +// if (store < 0) store = 0; +// if (store > 63) store = 63; // safety clamp +// metaWriter.writeBits(store, 5); +// } + } + + byte[] metaBytes = metaWriter.finish(); + dos.write(metaBytes); + dos.flush(); + + // --- Data bitstream --- + BitWriter dataWriter = new BitWriter(); + + // For each pack, find packMaxBitWidth and write each group's pack_size values using packMaxBitWidth bits + for ( Pack pack : packs) { + int packMaxBW = pack.maxBitWidth; + // no change for packMaxBW == 64: BitWriter supports splitting 64 into two 32-bit writes + for (int i = 0; i < pack.size; ++i) { + int originalGroupIndex = pack.indices.get(0); + int startPos = originalGroupIndex * pack_size; + for (int j = 0; j < pack_size; ++j) { + long val; +// if (startPos + j < dataArray.length) { + val = dataArray[startPos + j]; +// } else { +// val = 0L; +// } + // mask value to packMaxBW bits (if packMaxBW == 64, mask preserves full 64 bits) + long mask; + if (packMaxBW == 0) { + dataWriter.writeBits(0L, 0); // nothing to write + } else { + if (packMaxBW == 64) { + // write full 64-bit value (BitWriter handles split) + dataWriter.writeBits(val, 64); + } else { + mask = (1L << packMaxBW) - 1L; + long masked = val & mask; + dataWriter.writeBits(masked, packMaxBW); + } + } + } + } + } + + byte[] dataBytes = dataWriter.finish(); + dos.write(dataBytes); + dos.flush(); + + return baos.toByteArray(); + } + + // ========== packOctads (updated to accept originalLength for compression) ========== + static PackingResult packOctads(List bitWidths, RLDecisionModel model, List< DecisionPoint> decisionTrace, int pack_size, long[] dataArray, int originalLength) { + PackingResult result = new PackingResult(); + Pack currentPack = new Pack(); + int globalMaxLog = 0; + int packCount = 0; + + Random localRng = ThreadLocalRandom.current(); + + for (int i = 0; i < bitWidths.size(); ++i) { + int b = bitWidths.get(i); + + if (currentPack.size == 0) { + currentPack.addOctad(i, b); + } else if (b == currentPack.maxBitWidth) { + currentPack.addOctad(i, b); + } else { + float[] feat = new float[INPUT_DIM]; + feat[0] = currentPack.size / 100.0f; + feat[1] = currentPack.maxBitWidth / 64.0f; + feat[2] = b / 64.0f; + feat[3] = packCount / 100.0f; + feat[4] = globalMaxLog / 10.0f; + + float probability = model.forwardProb(feat); + + boolean shouldMerge; + if (localRng.nextFloat() < model.explorationRate) { + shouldMerge = (localRng.nextFloat() > 0.5f); + } else { + shouldMerge = probability > 0.5f; + } + + if (decisionTrace != null) { + decisionTrace.add(new DecisionPoint(currentPack.size, currentPack.maxBitWidth, b, packCount, globalMaxLog, shouldMerge, probability)); + } + + if (shouldMerge) { + currentPack.addOctad(i, b); + } else { + result.dataCostA += currentPack.dataCost(pack_size); + int logSize = currentPack.logSize(); + if (logSize > globalMaxLog) globalMaxLog = logSize; + result.packs.add(currentPack); + packCount++; + + currentPack = new Pack(); + currentPack.addOctad(i, b); + } + } + } + + if (currentPack.size > 0) { + result.dataCostA += currentPack.dataCost(pack_size); + int logSize = currentPack.logSize(); + if (logSize > globalMaxLog) globalMaxLog = logSize; + result.packs.add(currentPack); + packCount++; + } + + result.packCount = packCount; + result.calculateCost(globalMaxLog); + + // 执行实际的bitpacking压缩（如果提供了 dataArray） + if (dataArray != null) { + try { + result.compressedData = performBitPackingCompression64_fast(dataArray, result.packs, pack_size, originalLength); + } catch (IOException e) { + System.err.println("Compression failed: " + e.getMessage()); + result.compressedData = null; + } + } + + return result; + } + +// public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { +// try { +// // 这里需要根据实际的压缩格式来解析 +// // 假设compressedData包含bit-packed数据 +// ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); +// DataInputStream dis = new DataInputStream(bais); +// +// // 读取bit-packed数据 +// int totalGroups = bitWidths.length; +// long[] result = new long[totalGroups * packSize]; +// int resultIndex = 0; +// +// BitReader reader = new BitReader(compressedData); +// +// for (int g = 0; g < totalGroups; ++g) { +// int bw = bitWidths[g]; +// for (int k = 0; k < packSize; ++k) { +// if (bw == 0) { +// result[resultIndex++] = 0L; +// } else if (bw == 64) { +// long high = reader.readBits(32); +// long low = reader.readBits(32); +// long v = (high << 32) | (low & 0xFFFFFFFFL); +// result[resultIndex++] = v; +// } else { +// long v = reader.readBits(bw); +// result[resultIndex++] = v; +// } +// } +// } +// +// // 只取原始长度的数据并Sprintz解码 +// return Arrays.copyOf(result, originalLength); +// +// } catch (IOException e) { +// System.err.println("Fast decompression failed: " + e.getMessage()); +// return new long[0]; +// } +// } + + public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { + if (compressedData == null || compressedData.length == 0) { + System.err.println("Compressed data is null or empty"); + return new long[0]; + } + + try { + ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); + DataInputStream dis = new DataInputStream(bais); + + // === Header === + int totalPacks = dis.readUnsignedByte(); + int bitsForCount = dis.readUnsignedByte(); + + // === Read meta bitstream === + int metaStartOffset = 2; // two bytes read + BitReader metaReader = new BitReader(compressedData, metaStartOffset); + + // 解析每个pack的信息 + List packInfos = new ArrayList<>(); + int totalValuesToDecode = 0; + + for (int p = 0; p < totalPacks; ++p) { + int octadCount = (int) metaReader.readBits(bitsForCount); + int packBitWidth = (int) metaReader.readBits(6); + if (packBitWidth == 63) packBitWidth = 64; + + packInfos.add(new PackInfo(octadCount, packBitWidth)); + totalValuesToDecode += octadCount * packSize; + } + + // 计算数据部分的起始位置 + int metaBitsUsed = metaReader.consumedBits(); + int dataStartByte = metaStartOffset + (metaBitsUsed + 7) / 8; + + // 检查数据起始位置是否超出压缩数据范围 + if (dataStartByte >= compressedData.length) { +// System.err.println("Data start position exceeds compressed data length"); + return new long[0]; + } + + // === Data bitstream === + BitReader dataReader = new BitReader(compressedData, dataStartByte); + List resultList = new ArrayList<>(); + + // === 按pack解码数据 === + for ( PackInfo packInfo : packInfos) { + int octadCount = packInfo.octadCount; + int bitWidth = packInfo.bitWidth; + + // 检查剩余数据是否足够 + if (dataReader.remainingBits() < (long) octadCount * packSize * bitWidth) { +// System.err.println("Insufficient data for decoding pack. Expected: " + +// (octadCount * packSize * bitWidth) + " bits, Available: " + +// dataReader.remainingBits() + " bits"); + break; + } + + // 每个octad包含packSize个值 + for (int i = 0; i < octadCount; ++i) { + for (int j = 0; j < packSize; ++j) { + long value; + if (bitWidth == 0) { + value = 0L; + } else if (bitWidth == 64) { + // 64位特殊处理：分成两个32位读取 + long high = dataReader.readBits(32); + long low = dataReader.readBits(32); + value = (high << 32) | low; + } else { + value = dataReader.readBits(bitWidth); + } + resultList.add(value); + } + } + } + + // 转换为数组并截取到原始长度 + long[] result = new long[Math.min(resultList.size(), originalLength)]; + for (int i = 0; i < result.length; i++) { + result[i] = resultList.get(i); + } + +// System.out.println("Decompression completed: " + result.length + " values decoded"); + return result; + + } catch (Exception e) { + System.err.println("Fast decompression failed: " + e.getMessage()); + e.printStackTrace(); + return new long[0]; + } + } + + // 辅助类，存储pack信息 + static class PackInfo { + int octadCount; + int bitWidth; + + PackInfo(int octadCount, int bitWidth) { + this.octadCount = octadCount; + this.bitWidth = bitWidth; + } + } + // ========== Training loop (trainModel) ========== + static RLDecisionModel trainModel(int epochs, String csvFilePath) { + System.err.println("Training RL model from CSV data..."); + RLDecisionModel model = new RLDecisionModel(); + List> sequences = loadDataFromCSV(csvFilePath); + if (sequences.isEmpty()) { + System.err.println("No data loaded from CSV. Returning initial model."); + return model; + } + System.err.println("Loaded " + sequences.size() + " sequences from CSV"); + + List decisionTrace = new ArrayList<>(); + for (int epoch = 1; epoch <= epochs; ++epoch) { + long startTime = System.nanoTime(); + float totalReward = 0.0f; + float totalLoss = 0.0f; + int processedSequences = 0; + + for (List bitWidths : sequences) { + decisionTrace.clear(); + // training does not perform actual compression, pass dataArray=null and originalLength=0 + PackingResult result = packOctads(bitWidths, model, decisionTrace, 8, null, 0); + + float reward = (float) result.totalCost / 500000.0f; + totalReward += reward; + + float loss = model.train(decisionTrace, reward); + totalLoss += loss; + processedSequences++; + } + + long durationMs = (System.nanoTime() - startTime) / 1_000_000L; + + if (epoch % 10 == 0 || epoch == 1 || epoch == epochs) { + System.out.printf("Epoch %d: Avg Reward = %.6f, Avg Loss = %.6f, Time = %d ms%n", + epoch, + totalReward / processedSequences, + totalLoss / processedSequences, + durationMs); + } else { + System.out.printf("Epoch %d done. Time = %d ms%n", epoch, durationMs); + } + } + + return model; + } + // ========== performanceTest (optimized internals) ========== + static void performanceTestPack8(RLDecisionModel model, String directory, String outputDirStr) { + System.out.println("\nPerformance Testing..."); + Path outdir = Paths.get(outputDirStr); + try { + if (!Files.exists(outdir)) Files.createDirectories(outdir); + } catch (IOException e) { + System.err.println("Cannot create output dir: " + outputDirStr); + return; + } + + try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { + for (Path entry : ds) { + if (!Files.isRegularFile(entry)) continue; + String fname = entry.getFileName().toString(); + if (IGNORE_FILES.contains(fname)) continue; + + System.out.println("Processing " + fname + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + + try (BufferedReader br = Files.newBufferedReader(entry)) { + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(","); + for (String token : tokens) { + String t = trimStr(token); + if (!t.isEmpty()) { + numbers.add(t); + int dec = 0; + int pos = t.indexOf('.'); + if (pos != -1) dec = t.length() - pos - 1; + decimalPlaces.add(dec); + } + } + } + } catch (IOException e) { + System.err.println("Cannot open " + entry.toString()); + continue; + } + + if (numbers.isEmpty()) continue; + + Path outPath = outdir.resolve(fname); + try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { + writer.write("Input Direction,Encoding Algorithm,Encoding Time,Decoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); + + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + long modelCost = 0; + long modelTime = 0; + long compressedSize = 0; + long modelDecodeTime = 0; + + for (int rep = 0; rep < time_of_repeat; ++rep) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(numbers.size(), i + CHUNK_SIZE); + if (end - i <= 2) continue; + List chunkNumbers = numbers.subList(i, end); + int decimalMax = 0; + for (int k = i; k < end; ++k) { + if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); + } + + long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + long[] scaledInts = sprintz(scaledInt); + + int remainder = scaledInts.length % pack_size; + int padding = (remainder == 0) ? 0 : pack_size - remainder; + long[] padded = new long[scaledInts.length + padding]; + System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); + if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); + + int groups = padded.length / pack_size; + int[] bitWidths = new int[groups]; + int gidx = 0; + for (int si = 0; si < padded.length; si += pack_size) { + long maxInGroup = 0; + for (int sj = si; sj < si + pack_size; ++sj) { + long v = padded[sj]; + if (v > maxInGroup) maxInGroup = v; + } + int bitWidth = 0; + if (maxInGroup > 0) { + bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + } else { + bitWidth = 0; + } + bitWidths[gidx++] = bitWidth; + } + + // pass original length (un-padded) so decoder can trim + List bitWidthsList = new ArrayList<>(groups); + for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); + + PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); + long duration = System.nanoTime() - startTime; + + + // 使用快速解压 + if (res.compressedData != null) { + long decodeStartTime = System.nanoTime(); + long[] decompressed = fastDecompress(res.compressedData, bitWidths, pack_size, scaledInts.length); + long[] decompressed_final = sprintzDecode(decompressed); + long decodeDuration = System.nanoTime() - decodeStartTime; + modelDecodeTime += decodeDuration; +// System.out.println(decodeDuration); + } + + modelTime += duration; + modelCost += (res.compressedData.length* 8L); + + } + } + + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + modelDecodeTime /= time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms + double decodeThroughput = (double) (numbers.size() * 8000) / modelDecodeTime; // points per second + writer.write(entry.toString() + ","); + writer.write("SPRINTZ-RL,"); + writer.write(String.valueOf(modelTime_throughput) + ","); + writer.write(String.valueOf(decodeThroughput) + ","); + writer.write(String.valueOf(numbers.size()) + ","); + writer.write(String.valueOf(modelCost) + ","); + writer.write(String.valueOf(pack_size) + ","); + writer.write(String.valueOf(model_ratio) + "\n"); + } + } catch (IOException e) { + System.err.println("Error writing output file for " + fname); + } + } + } catch (IOException e) { + System.err.println("Error iterating directory: " + directory); + } + } + + public static void main(String[] args) { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv";// args.length > 0 ? args[0] : ""; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel";//args.length > 1 ? args[1] : ""; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_rl";// args.length > 2 ? args[2] : "./output_BPRL"; + + int epochs = 20; + + if (args.length >= 1) trainCsv = args[0]; + if (args.length >= 2) dataDir = args[1]; + if (args.length >= 3) outDir = args[2]; + + RLDecisionModel model = new RLDecisionModel(); + if (!trainCsv.isEmpty()) { + model = trainModel(epochs, trainCsv); + } else { + System.err.println("No training CSV given. Using randomly initialized RL model."); + } + + if (!dataDir.isEmpty()) { + performanceTestPack8(model, dataDir, outDir); + } else { + System.err.println("No data directory provided for performanceTest. Exiting."); + } + } + + + // ========== performanceTest (optimized internals) ========== + static void performanceTest(RLDecisionModel model, String directory, String outputDirStr) { + System.out.println("\nPerformance Testing..."); + Path outdir = Paths.get(outputDirStr); + try { + if (!Files.exists(outdir)) Files.createDirectories(outdir); + } catch (IOException e) { + System.err.println("Cannot create output dir: " + outputDirStr); + return; + } + + try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { + for (Path entry : ds) { + if (!Files.isRegularFile(entry)) continue; + String fname = entry.getFileName().toString(); + if (IGNORE_FILES.contains(fname)) continue; + + System.out.println("Processing " + fname + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + + try (BufferedReader br = Files.newBufferedReader(entry)) { + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(","); + for (String token : tokens) { + String t = trimStr(token); + if (!t.isEmpty()) { + numbers.add(t); + int dec = 0; + int pos = t.indexOf('.'); + if (pos != -1) dec = t.length() - pos - 1; + decimalPlaces.add(dec); + } + } + } + } catch (IOException e) { + System.err.println("Cannot open " + entry.toString()); + continue; + } + + if (numbers.isEmpty()) continue; + + Path outPath = outdir.resolve(fname); + try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { + writer.write("Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); + + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 9; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + System.out.println(pack_size); + long modelCost = 0; + long modelTime = 0; + long compressedSize = 0; + + for (int rep = 0; rep < time_of_repeat; ++rep) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(numbers.size(), i + CHUNK_SIZE); + if (end - i <= 2) continue; + List chunkNumbers = numbers.subList(i, end); + int decimalMax = 0; + for (int k = i; k < end; ++k) { + if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); + } + + long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + long[] scaledInts = sprintz(scaledInt); + + int remainder = scaledInts.length % pack_size; + int padding = (remainder == 0) ? 0 : pack_size - remainder; + long[] padded = new long[scaledInts.length + padding]; + System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); + if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); + + int groups = padded.length / pack_size; + int[] bitWidths = new int[groups]; + int gidx = 0; + for (int si = 0; si < padded.length; si += pack_size) { + long maxInGroup = 0; + for (int sj = si; sj < si + pack_size; ++sj) { + long v = padded[sj]; + if (v > maxInGroup) maxInGroup = v; + } + int bitWidth = 0; + if (maxInGroup > 0) { + bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + } else { + bitWidth = 0; + } + bitWidths[gidx++] = bitWidth; + } + + // pass original length (un-padded) so decoder can trim + List bitWidthsList = new ArrayList<>(groups); + for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); + + PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += (res.compressedData.length*8); + +// if (rep == 0) { +// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; +// } + } + } + + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms + + writer.write(entry.toString() + ","); + writer.write("SPRINTZ-RL,"); + writer.write(String.valueOf(modelTime_throughput) + ","); + writer.write(String.valueOf(numbers.size()) + ","); + writer.write(String.valueOf(modelCost) + ","); + writer.write(String.valueOf(pack_size) + ","); + writer.write(String.valueOf(model_ratio) + "\n"); + } + } catch (IOException e) { + System.err.println("Error writing output file for " + fname); + } + } + } catch (IOException e) { + System.err.println("Error iterating directory: " + directory); + } + } + @Test + public void TestVarPackSize() { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv";// args.length > 0 ? args[0] : ""; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel";//args.length > 1 ? args[1] : ""; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_RL_vary_pack_size";// args.length > 2 ? args[2] : "./output_BPRL"; + + int epochs = 20; + + RLDecisionModel model = new RLDecisionModel(); + model = trainModel(epochs, trainCsv); + performanceTest(model, dataDir, outDir); + } + + + static void performanceTestVariableChunkSize(RLDecisionModel model, String directory, String outputDirStr) { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + Path outdir = Paths.get(outputDirStr); + try { + if (!Files.exists(outdir)) Files.createDirectories(outdir); + } catch (IOException e) { + System.err.println("Cannot create output dir: " + outputDirStr); + return; + } + + // Define the chunk sizes to test (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { + for (Path entry : ds) { + if (!Files.isRegularFile(entry)) continue; + String fname = entry.getFileName().toString(); + if (IGNORE_FILES.contains(fname)) continue; + + System.out.println("Processing " + fname + " with variable chunk sizes..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + + try (BufferedReader br = Files.newBufferedReader(entry)) { + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split(","); + for (String token : tokens) { + String t = trimStr(token); + if (!t.isEmpty()) { + numbers.add(t); + int dec = 0; + int pos = t.indexOf('.'); + if (pos != -1) dec = t.length() - pos - 1; + decimalPlaces.add(dec); + } + } + } + } catch (IOException e) { + System.err.println("Cannot open " + entry.toString()); + continue; + } + + if (numbers.isEmpty()) continue; + + Path outPath = outdir.resolve(fname.replace(".", "_chunksize_test.")); + try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { + writer.write("m,Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); + + int time_of_repeat = 50; // Reduced for faster testing with multiple chunk sizes + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + long[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + long[] scaledInts_all = new long[totalLength]; + + int currentIndex = 0; + for (long[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // Test each chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for (int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + long modelCost = 0; + long modelTime = 0; + long compressedSize = 0; + + for (int rep = 0; rep < time_of_repeat; ++rep) { + for (int i = 0; i < numbers.size(); i += chunkSize) { +// int end = Math.min(numbers.size(), i + chunkSize); +// if (end - i <= 2) continue; +// List chunkNumbers = numbers.subList(i, end); +// int decimalMax = 0; +// for (int k = i; k < end; ++k) { +// if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); +// } +// +// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + int end = Math.min(i + chunkSize, numbers.size()); + long[] scaledInt = new long[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + long[] scaledInts = sprintz(scaledInt); + + int remainder = scaledInts.length % pack_size; + int padding = (remainder == 0) ? 0 : pack_size - remainder; + long[] padded = new long[scaledInts.length + padding]; + System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); + if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); + + int groups = padded.length / pack_size; + int[] bitWidths = new int[groups]; + int gidx = 0; + for (int si = 0; si < padded.length; si += pack_size) { + long maxInGroup = 0; + for (int sj = si; sj < si + pack_size; ++sj) { + long v = padded[sj]; + if (v > maxInGroup) maxInGroup = v; + } + int bitWidth = 0; + if (maxInGroup > 0) { + bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + } else { + bitWidth = 0; + } + bitWidths[gidx++] = bitWidth; + } + + // pass original length (un-padded) so decoder can trim + List bitWidthsList = new ArrayList<>(groups); + for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); + + PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += (res.compressedData.length* 8L); + + } + } + + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms + + writer.write(String.valueOf(chunkSize/8) + ","); + writer.write(entry.toString() + ","); + writer.write("sprintz-RL,"); + writer.write(String.valueOf(modelTime_throughput) + ","); + writer.write(String.valueOf(numbers.size()) + ","); + writer.write(String.valueOf(modelCost) + ","); + writer.write(String.valueOf(pack_size) + ","); + writer.write(String.valueOf(model_ratio) + "\n"); + } + } + } catch (IOException e) { + System.err.println("Error writing output file for " + fname); + } + } + } catch (IOException e) { + System.err.println("Error iterating directory: " + directory); + } + } + @Test + public void TestVariableChunkSize() { + String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprint_RL_vary_m"; + + int epochs = 20; + + RLDecisionModel model = new RLDecisionModel(); + model = trainModel(epochs, trainCsv); + performanceTestVariableChunkSize(model, dataDir, outDir); + } + +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java new file mode 100644 index 000000000..bd1c8457f --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java @@ -0,0 +1,798 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + + +public class FBitpacking512 { + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + private static final int CHUNK_SIZE = 1024; + + public static int getBitWith(int num) { + if (num == 0) + return 1; + else + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static int getCount(long long1, int mask) { + return ((int) (long1 & mask)); + } + + public static int getUniqueValue(long long1, int left_shift) { + return ((int) ((long1) >> left_shift)); + } + + public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer); + } + + private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + if (num > 4) { + System.out.println("bytes2Integer error"); + return 0; + } + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + private static long bytesLong2Integer(byte[] encoded, int decode_pos) { + long value = 0; + for (int i = 0; i < 4; i++) { + value <<= 8; + int b = encoded[i + decode_pos] & 0xFF; + value |= b; + } + return value; + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + // remaining bits for the current unfinished Integer + int leftBit = 0; + + while (valueIdx < 8 + offset) { + // buffer is used for saving 32 bits as a part of result + int buffer = 0; + // remaining size of bits in the 'buffer' + int leftSize = 32; + + // encode the left bits of current Integer to 'buffer' + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + // encode one Integer to the 'buffer' + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + // If the remaining space of the buffer can not save the bits for one Integer, + if (leftSize > 0 && valueIdx < 8 + offset) { + // put the first 'leftSize' bits of the Integer into remaining space of the + // buffer + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + // put the buffer into the final result + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + // total bits which have read from 'buf' to 'buffer'. i.e., + // number of available bits to be decoded. + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + // If current available bits are not enough to decode one Integer, + // then add next byte from buf to 'buffer' until totalBits >= width + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + // If current available bits are enough to decode one Integer, + // then decode one Integer one by one until left bits in 'buffer' is + // not enough to decode one Integer. + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + return encode_pos; + + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + // 新增的解压函数 +// public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { +// List result = new ArrayList<>(); +// int decodePos = 0; +// +// for (int group = 0; group < bitWidths.length; group++) { +// int bitWidth = compressedData[decodePos++] & 0xFF; +// +// // 解压当前分组 +// ArrayList groupData = new ArrayList<>(); +// unpack8Values(compressedData, decodePos, bitWidth, groupData); +// +// // 添加解压出的数据 +// for (int i = 0; i < pack_size; i++) { +// if (result.size() < originalLength) { +// result.add(groupData.get(i)); +// } +// } +// +// decodePos += bitWidth; +// } +// +// // 转换为数组返回 +// int[] decodedArray = new int[result.size()]; +// for (int i = 0; i < result.size(); i++) { +// decodedArray[i] = result.get(i); +// } +// return decodedArray; +// } + + public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { + int[] result = new int[originalLength]; // 直接使用数组 + int resultIndex = 0; + int decodePos = 0; + + for (int group = 0; group < bitWidths.length && resultIndex < originalLength; group++) { + int bitWidth = compressedData[decodePos++] & 0xFF; + + // 预分配固定大小的列表 + ArrayList groupData = new ArrayList<>(pack_size); + unpack8Values(compressedData, decodePos, bitWidth, groupData); + + // 批量拷贝 + int copyLength = Math.min(pack_size, originalLength - resultIndex); + for (int i = 0; i < copyLength; i++) { + result[resultIndex++] = groupData.get(i); + } + + decodePos += bitWidth; + } + + return result; + } + + // 新增的完整解压函数（包含头部信息解析） + public static int[] decodeBitPackingWithHeader(byte[] encodedWithHeader, int pack_size) { + // 解析头部信息 - 假设前4个字节存储原始数据长度 + int originalLength = bytes2Integer(encodedWithHeader, 0, 4); + + // 解析分组数 + int groupCount = bytes2Integer(encodedWithHeader, 4, 4); + + // 解析位宽数组 + int[] bitWidths = new int[groupCount]; + int headerSize = 8; // 4字节原始长度 + 4字节分组数 + for (int i = 0; i < groupCount; i++) { + bitWidths[i] = encodedWithHeader[headerSize + i] & 0xFF; + } + + // 压缩数据起始位置 + int dataStart = headerSize + groupCount; + byte[] compressedData = new byte[encodedWithHeader.length - dataStart]; + System.arraycopy(encodedWithHeader, dataStart, compressedData, 0, compressedData.length); + + // 调用解压函数 + return decodeBitPacking(compressedData, bitWidths, pack_size, originalLength); + } + + private static int[] scaleNumbers(List numbers, int decimalMax) { + // 1. 预先计算缩放因子 + BigDecimal scale = BigDecimal.TEN.pow(decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + + if (size == 0) { + return result; + } + + // 2. 单次遍历完成所有转换和最小值查找 + BigDecimal min = null; + BigDecimal[] scaledValues = new BigDecimal[size]; + + for (int i = 0; i < size; i++) { + BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); + scaledValues[i] = val; + if (min == null || val.compareTo(min) < 0) { + min = val; + } + } + + // 3. 处理第一个元素 + BigDecimal first = scaledValues[0].subtract(min); + result[0] = first.toBigInteger().intValue(); + + // 4. 处理后续元素（差分+ZigZag） + for (int i = 1; i < size; i++) { + BigDecimal current = scaledValues[i].subtract(min); + result[i]=current.toBigInteger().intValue(); + } + + return result; + } + + public static void main(String[] args) throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + // 更新表头，增加解压吞吐率列 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 500; + + + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; // 新增：解压时间 + + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + // 1. 找出当前8个元素中的最大值 + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / 8] = bitWidth; + } + +// int fixed_block = CHUNK_SIZE/40; + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, 8); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 新增：测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPacking(compressedData, bitWidths, 8, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + +// // 可选：验证解压数据的正确性（只在第一次重复时验证） +// if (j == 0) { +// boolean correct = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// correct = false; +// System.err.println("Decompression error at index " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// break; +// } +// } +//// if (correct) { +//// System.out.println("Decompression verified successfully for chunk " + (i/CHUNK_SIZE)); +//// } +// } + } + + } + modelCost = modelCost/time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + modelDecodeTime = (modelDecodeTime)/time_of_repeat; // 平均解压时间 + + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000L)/ (double) (modelTime); + double modelDecodeTime_throughput = (double)(numbers.size()*8000L)/ (double) (modelDecodeTime); + + // 更新输出记录，包含解压吞吐率 + String[] record = { + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + + } + + public static byte[] encodeBitPacking(int[] paddedArray, int[] bitWidths, int pack_size) { + List result = new ArrayList<>(); + + int totalGroups = bitWidths.length; + int max_bit_width = 0; + + for (int i = 0; i < totalGroups; i++) { + if (bitWidths[i] > max_bit_width) { + max_bit_width = bitWidths[i]; + } + } + int totalBitPackedBytes = (max_bit_width*pack_size*totalGroups+7)/8; + byte[] bitPackedData = new byte[totalBitPackedBytes +totalGroups+ 32]; + int encodePos = 0; + + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + ArrayList groupData = new ArrayList<>(); + for (int i = 0; i < pack_size; i++) { + if (startIndex + i < paddedArray.length) { + groupData.add(paddedArray[startIndex + i]); + } else { + groupData.add(0); + } + } + + bitPackedData[encodePos++] = (byte) bitWidths[group]; + encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); + } + + for (int i = 0; i < encodePos; i++) { + result.add(bitPackedData[i]); + } + + byte[] finalResult = new byte[result.size()]; + for (int i = 0; i < result.size(); i++) { + finalResult[i] = result.get(i); + } + + return finalResult; + } + + public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { + int blocksize= bitWidths.length; + int totalCost = 0; + int numPacks = (int) Math.ceil((double) blocksize / fixed_pack); + + for (int pack = 0; pack < numPacks; pack++) { + int start = pack * fixed_pack; + int end = Math.min(start + fixed_pack, blocksize); + + int maxBitWidth = 0; + for (int i = start; i < end; i++) { + if (bitWidths[i] > maxBitWidth) { + maxBitWidth = bitWidths[i]; + } + } + + totalCost += pack_size * (end-start) * maxBitWidth; + } + + totalCost += 5 * blocksize / fixed_pack; + return totalCost; + } + + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP_vary_pack_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + // 方法：强化学习 + int modelCost = 0; + long modelTime = 0; + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + String[] record = { + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + writer.close(); + } + + } + + // 新增方法：测试不同chunk size的表现 + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为pack_size的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; +// if(i==0){ +// System.out.println(Arrays.toString(paddedArray)); +// } + + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; +// System.out.println(bitWidth); + } + + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java new file mode 100644 index 000000000..7280298ae --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java @@ -0,0 +1,1158 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +//import org.openjdk.jol.info.ClassLayout; +//import org.openjdk.jol.info.GraphLayout; + +public class FSprintz512 { + private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv", + "POI-lon.csv","Air-sensor.csv","Basel-wind.csv","Basel-temp.csv"); +// private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv"); + private static final int CHUNK_SIZE = 1024; + // 轻量级Octad表示 + + public static int getBitWith(int num) { + if (num == 0) + return 1; + else + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static int getCount(long long1, int mask) { + return ((int) (long1 & mask)); + } + + public static int getUniqueValue(long long1, int left_shift) { + return ((int) ((long1) >> left_shift)); + } + + public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer); + } + + private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + if (num > 4) { + System.out.println("bytes2Integer error"); + return 0; + } + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + private static long bytesLong2Integer(byte[] encoded, int decode_pos) { + long value = 0; + for (int i = 0; i < 4; i++) { + value <<= 8; + int b = encoded[i + decode_pos] & 0xFF; + value |= b; + } + return value; + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + // remaining bits for the current unfinished Integer + int leftBit = 0; + + while (valueIdx < 8 + offset) { + // buffer is used for saving 32 bits as a part of result + int buffer = 0; + // remaining size of bits in the 'buffer' + int leftSize = 32; + + // encode the left bits of current Integer to 'buffer' + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + // encode one Integer to the 'buffer' + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + // If the remaining space of the buffer can not save the bits for one Integer, + if (leftSize > 0 && valueIdx < 8 + offset) { + // put the first 'leftSize' bits of the Integer into remaining space of the + // buffer + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + // put the buffer into the final result + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + // total bits which have read from 'buf' to 'buffer'. i.e., + // number of available bits to be decoded. + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + // If current available bits are not enough to decode one Integer, + // then add next byte from buf to 'buffer' until totalBits >= width + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + // If current available bits are enough to decode one Integer, + // then decode one Integer one by one until left bits in 'buffer' is + // not enough to decode one Integer. + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + return encode_pos; + + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + private static int zigzagEncode(int n) { + return (n << 1) ^ (n >> 31); + } + public static int[] scaleNumbers(List numbers, int decimalMax) { + int scale = (int) Math.pow(10, decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + + if (size == 0) { + return result; + } + + // 1. Parse all numbers and scale them up + int[] scaledValues = new int[size]; + for (int i = 0; i < size; i++) { + String numStr = numbers.get(i); + // Parse the number (handling both "123.456" and "123" cases) + String[] parts = numStr.split("\\."); + int whole = Integer.parseInt(parts[0]); + + // Handle fractional part + int fraction = 0; + if (parts.length > 1) { + String fractionStr = parts[1]; + // Pad with zeros if necessary to ensure proper scaling + if (fractionStr.length() < decimalMax) { + while (fractionStr.length() < decimalMax) { + fractionStr += "0"; + } + } else if (fractionStr.length() > decimalMax) { + // Truncate if too many decimal places (alternative could be rounding) + fractionStr = fractionStr.substring(0, decimalMax); + } + fraction = Integer.parseInt(fractionStr); + } + + scaledValues[i] = whole * scale + fraction; + } + +// // 2. Process first element +// int first = scaledValues[0]; +// result[0] = first; +// +// // 3. Process subsequent elements with delta + ZigZag encoding +// int prev = first; +// for (int i = 1; i < size; i++) { +// int current = scaledValues[i]; +// int diff = current - prev; +// result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding +// prev = current; +// } + + return scaledValues; + } + + public static int[] sprintz(int[] numbers) { + int size = numbers.length; + int[] result = new int[size]; + + int first = numbers[0]; + result[0] = first; + + // 3. Process subsequent elements with delta + ZigZag encoding + int prev = first; + for (int i = 1; i < size; i++) { + int current = numbers[i]; + int diff = current - prev; + result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding + prev = current; + } + + return result; + } + + public static byte[] encodeBitPacking(int[] paddedArray, int[] bitWidths, int pack_size) { + List result = new ArrayList<>(); + + + + // 3. 对paddedArray进行bit-packing + int totalGroups = bitWidths.length; + + // 计算bit-packed数据的总字节数 - 修正计算方式 +// int totalBitPackedBytes = (cost_bits+7)/8; +// for (int i = 0; i < totalGroups; i++) { +// // 每组需要 ceil(8 * bitWidth / 8) = bitWidth 字节 +// totalBitPackedBytes += bitWidths[i]; +// } + + // 确保数组足够大，添加一些额外空间以防万一 + int max_bit_width = 0; + + for (int bitWidth : bitWidths) { + if (bitWidth > max_bit_width) { + max_bit_width = bitWidth; + } + } +// System.out.println(max_bit_width); +// if(max_bit_width ==0) { +// System.out.println(Arrays.toString(bitWidths)); +// } +// System.out.println(pack_size); +// System.out.println(totalGroups); + int totalBitPackedBytes = (max_bit_width*pack_size*totalGroups+7)/8; + byte[] bitPackedData = new byte[totalBitPackedBytes + totalGroups+32]; +// bitPackedData[0] = (byte) max_bit_width; + int encodePos = 0; + // 对每组数据进行bit-packing + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + ArrayList groupData = new ArrayList<>(); + for (int i = 0; i < pack_size; i++) { + if (startIndex + i < paddedArray.length) { + groupData.add(paddedArray[startIndex + i]); + } else { + groupData.add(0); // 用0填充不足的部分 + } + } + + bitPackedData[encodePos++] = (byte) bitWidths[group]; + encodePos = bitPacking(groupData, 0,bitWidths[group] , encodePos, bitPackedData); + } + + + // 4. 将bit-packed数据写入结果（只写入实际使用的部分） + for (int i = 0; i < encodePos; i++) { + result.add(bitPackedData[i]); + } + + // 转换为byte数组返回 + byte[] finalResult = new byte[result.size()]; + for (int i = 0; i < result.size(); i++) { + finalResult[i] = result.get(i); + } + + return finalResult; + } + public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { + int blocksize= bitWidths.length; +// int minCost = Integer.MAX_VALUE; + + // Try all possible pack sizes from 1 to CHUNK_SIZE +// for (int p = 1; p <= blocksize; p++) { + int totalCost = 0; + int numPacks = (int) Math.ceil((double) blocksize / fixed_pack); + + // Calculate cost for each pack + for (int pack = 0; pack < numPacks; pack++) { + int start = pack * fixed_pack; + int end = Math.min(start + fixed_pack, blocksize); + + + // Find max bitWidth in current pack + int maxBitWidth = 0; + for (int i = start; i < end; i++) { + if (bitWidths[i] > maxBitWidth) { + maxBitWidth = bitWidths[i]; + } + } + + // Add to cost: 8 * p * maxBitWidth + totalCost += pack_size * (end-start) * maxBitWidth; + } + + // Add the chunk cost: 5 * CHUNK_SIZE / p + totalCost += 5 * blocksize / fixed_pack; + +// // Update minimum cost +// if (totalCost < minCost) { +// minCost = totalCost; +// } +// } + return totalCost; + } + + /** + * ZigZag解码 + */ + private static int zigzagDecode(int n) { + return (n >>> 1) ^ -(n & 1); + } + + /** + * 解压函数 - 从压缩数据中恢复原始整数数组 + */ +// public static int[] decodeBitPackingFull(byte[] compressedData, int originalLength, int packSize) { +// List decodedValues = new ArrayList<>(); +// int decodePos = 0; +// +// // 计算分组数量 +// int totalGroups = (originalLength + packSize - 1) / packSize; +// +// for (int group = 0; group < totalGroups; group++) { +// // 读取该组的位宽 +// int bitWidth = compressedData[decodePos++] & 0xFF; +// +// if (bitWidth == 0) { +// // 如果位宽为0，说明这组都是0 +// for (int i = 0; i < packSize; i++) { +// decodedValues.add(0); +// } +// continue; +// } +// +// // 解压该组数据 +// ArrayList groupData = new ArrayList<>(); +// unpack8Values(compressedData, decodePos, bitWidth, groupData); +// decodePos += bitWidth; +// +// // 添加到结果列表 +// for (int value : groupData) { +// decodedValues.add(value); +// } +// } +// +// // 转换为数组并截取原始长度（因为可能有填充） +// int[] result = new int[originalLength]; +// for (int i = 0; i < originalLength; i++) { +// result[i] = decodedValues.get(i); +// } +// +// return result; +// } + public static int[] decodeBitPackingFull(byte[] compressedData, int originalLength, int packSize) { + // 预分配结果数组，避免ArrayList的动态扩容开销 + int[] result = new int[originalLength]; + int resultIndex = 0; + int decodePos = 0; + + // 计算分组数量 + int totalGroups = (originalLength + packSize - 1) / packSize; + + // 预分配组数据数组，避免在循环中重复创建 + int[] groupData = new int[packSize]; + + for (int group = 0; group < totalGroups && resultIndex < originalLength; group++) { + // 读取该组的位宽 + int bitWidth = compressedData[decodePos++] & 0xFF; + + if (bitWidth == 0) { + // 如果位宽为0，说明这组都是0 - 直接填充0 + int fillCount = Math.min(packSize, originalLength - resultIndex); + // Arrays.fill比循环更快 + if (fillCount > 0) { + Arrays.fill(result, resultIndex, resultIndex + fillCount, 0); + resultIndex += fillCount; + } + continue; + } + + // 直接解压到预分配的groupData数组，避免ArrayList的开销 + int actualUnpacked = unpack8ValuesToArray(compressedData, decodePos, bitWidth, groupData); + decodePos += bitWidth; + + // 直接复制到结果数组，避免额外的循环 + int copyCount = Math.min(actualUnpacked, originalLength - resultIndex); + System.arraycopy(groupData, 0, result, resultIndex, copyCount); + resultIndex += copyCount; + } + + return result; + } + private static int unpack8ValuesToArray(byte[] encoded, int offset, int width, int[] result) { + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8 && valueIdx < result.length) { + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < 8 && valueIdx < result.length) { + result[valueIdx] = (int) (buffer >>> (totalBits - width)); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + + return valueIdx; // 返回实际解压的数量 + } + /** + * Sprintz解码 - 从差分编码恢复原始数据 + */ + public static int[] sprintzDecode(int[] encodedData) { + int size = encodedData.length; + int[] result = new int[size]; + + if (size == 0) return result; + + // 第一个元素是原始值 + result[0] = encodedData[0]; + + // 后续元素需要ZigZag解码和累加 + int prev = result[0]; + for (int i = 1; i < size; i++) { + int zigzagEncoded = encodedData[i]; + int diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 + result[i] = prev + diff; + prev = result[i]; + } + + return result; + } + public static double[] unscaleNumbers(int[] scaledValues, int decimalMax) { + double scale = Math.pow(10, decimalMax); + int size = scaledValues.length; + double[] result = new double[size]; + + for (int i = 0; i < size; i++) { + result[i] = scaledValues[i] / scale; + } + + return result; + } + public static int[] decompress(byte[] compressedData, int originalLength, int decimalMax, int packSize) { + // 1. 位解压 + int[] bitUnpacked = decodeBitPackingFull(compressedData, originalLength, packSize); + + // 2. Sprintz解码 + int[] sprintzDecoded = sprintzDecode(bitUnpacked); + + // 3. 缩放逆变换 +// double[] unscaled = unscaleNumbers(sprintzDecoded, decimalMax); + + return sprintzDecoded; + } + public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { + int[] result = new int[originalLength]; // 直接使用数组 + int resultIndex = 0; + int decodePos = 0; + + for (int group = 0; group < bitWidths.length && resultIndex < originalLength; group++) { + int bitWidth = compressedData[decodePos++] & 0xFF; + + // 预分配固定大小的列表 + ArrayList groupData = new ArrayList<>(pack_size); + unpack8Values(compressedData, decodePos, bitWidth, groupData); + + // 批量拷贝 + int copyLength = Math.min(pack_size, originalLength - resultIndex); + for (int i = 0; i < copyLength; i++) { + result[resultIndex++] = groupData.get(i); + } + + decodePos += bitWidth; + } + + return result; + } + + @Test + public void printDataTest() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); +// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/bitwidth"; + File outputDir = new File(outputDirstr); + +// RLDecisionModel trainedModel = trainModel(20, csvFilePath); + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; +// if(!file.getName().equals("Stocks-DE.csv")) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "BP", + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + List bitWidthRecords = new ArrayList<>(); + List sprintzBitWidthRecords = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + bitWidths[scaledInts_i / 8] = bitWidth; + String[] record = { + String.valueOf(bitWidth), + }; + writer.writeRecord(record); + } + + } + + + writer.close(); +// break; + } + } + + @Test + public void printSprintzDataTest() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); +// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/bitwidth_sprintz"; + File outputDir = new File(outputDirstr); + +// RLDecisionModel trainedModel = trainModel(20, csvFilePath); + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; +// if(!file.getName().equals("Stocks-DE.csv")) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Sprintz", + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + List bitWidthRecords = new ArrayList<>(); + List sprintzBitWidthRecords = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scalingInt); + + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); + + bitWidths[scaledInts_i / 8] = bitWidth; + String[] record = { + String.valueOf(bitWidth), + }; + writer.writeRecord(record); + } + + } + + + writer.close(); +// break; + } + } + + public static void main(String[] args) throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); +// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz"; + File outputDir = new File(outputDirstr); + +// RLDecisionModel trainedModel = trainModel(20, csvFilePath); + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; +// if(!file.getName().equals("Stocks-DE.csv")) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; +// System.out.println(numbers.size()); + + + // 方法：强化学习 +// long modelStart = System.nanoTime(); + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; // 新增解码时间统计 + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scalingInt); + + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + bitWidths[scaledInts_i / 8] = bitWidth; + } +// System.out.println(Arrays.toString(bitWidths)); + int fixed_pack = CHUNK_SIZE / 40; +// int cur_cost = computeMinPackingCost(bitWidths,fixed_pack, 8); + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, 8); + int cur_cost = compressedData.length * 8; // 转换为bit数 +// System.out.println(cur_cost); +// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 + long duration = System.nanoTime() - startTime; + + long decodeStartTime = System.nanoTime(); + // 执行解压 + int[] decodedData = decodeBitPacking(compressedData, bitWidths, 8, scaledInts.length); + sprintzDecode(decodedData); +// int[] decompressed = decompress(compressedData, chunkNumbers.size(), decimalMax, 8); + + long decodeDuration = System.nanoTime() - decodeStartTime; + modelDecodeTime += decodeDuration; + + modelTime += (duration); + modelCost += cur_cost; +// if(i==0) +// for (int episode = 0; episode < 10; episode++) { +// trainEpisode(scaledInts, episode); +// } +// List optimalK = predictOptimalK(scaledInts); +// System.out.println("Optimal k sequence: " + optimalK); + } + + } + modelCost /=time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); + double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); + String[] record = { + file.toString(), + "Sprintz", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); +// break; + } + } + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_SPRINTZ_vary_pack_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + // 方法：强化学习 + int modelCost = 0; + long modelTime = 0; + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + + List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + String[] record = { + file.toString(), + "SPRINTZ", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + writer.close(); + } + } + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为pack_size的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; +// if(i==0){ +// System.out.println(Arrays.toString(paddedArray)); +// } + + + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; +// System.out.println(bitWidth); + } + + byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java new file mode 100644 index 000000000..ed1d315c7 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java @@ -0,0 +1,916 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class RLEPackBitWidthSprintzTest { +// private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv", +// "POI-lon.csv","Air-sensor.csv","Basel-temp.csv"); +static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + private static final int CHUNK_SIZE = 1024; + + public static int getCount(long long1, int mask) { + return ((int) (long1 & mask)); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + if (num > 4) { + System.out.println("bytes2Integer error"); + return 0; + } + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + // remaining bits for the current unfinished Integer + int leftBit = 0; + + while (valueIdx < 8 + offset) { + // buffer is used for saving 32 bits as a part of result + int buffer = 0; + // remaining size of bits in the 'buffer' + int leftSize = 32; + + // encode the left bits of current Integer to 'buffer' + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + // encode one Integer to the 'buffer' + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + // If the remaining space of the buffer can not save the bits for one Integer, + if (leftSize > 0 && valueIdx < 8 + offset) { + // put the first 'leftSize' bits of the Integer into remaining space of the + // buffer + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + // put the buffer into the final result + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + // total bits which have read from 'buf' to 'buffer'. i.e., + // number of available bits to be decoded. + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + // If current available bits are not enough to decode one Integer, + // then add next byte from buf to 'buffer' until totalBits >= width + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + // If current available bits are enough to decode one Integer, + // then decode one Integer one by one until left bits in 'buffer' is + // not enough to decode one Integer. + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + return encode_pos; + + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + public static int[] scaleNumbers(List numbers, int decimalMax) { + int scale = (int) Math.pow(10, decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + + if (size == 0) { + return result; + } + + // 1. Parse all numbers and scale them up + int[] scaledValues = new int[size]; + for (int i = 0; i < size; i++) { + String numStr = numbers.get(i); + // Parse the number (handling both "123.456" and "123" cases) + String[] parts = numStr.split("\\."); + int whole = Integer.parseInt(parts[0]); + + // Handle fractional part + int fraction = 0; + if (parts.length > 1) { + String fractionStr = parts[1]; + // Pad with zeros if necessary to ensure proper scaling + if (fractionStr.length() < decimalMax) { + while (fractionStr.length() < decimalMax) { + fractionStr += "0"; + } + } else if (fractionStr.length() > decimalMax) { + // Truncate if too many decimal places (alternative could be rounding) + fractionStr = fractionStr.substring(0, decimalMax); + } + fraction = Integer.parseInt(fractionStr); + } + + scaledValues[i] = whole * scale + fraction; + } + return scaledValues; + } +// private static int[] scaleNumbers(List numbers, int decimalMax) { +// // 1. 预先计算缩放因子 +// BigDecimal scale = BigDecimal.TEN.pow(decimalMax); +// int size = numbers.size(); +// int[] result = new int[size]; +// +// if (size == 0) { +// return result; +// } +// +// // 2. 单次遍历完成所有转换和最小值查找 +// BigDecimal min = null; +// BigDecimal[] scaledValues = new BigDecimal[size]; +// +// for (int i = 0; i < size; i++) { +// BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); +// scaledValues[i] = val; +// if (min == null || val.compareTo(min) < 0) { +// min = val; +// } +// } +// +// // 3. 处理第一个元素 +// BigDecimal first = scaledValues[0].subtract(min); +// result[0] = first.toBigInteger().intValue(); +// +// // 4. 处理后续元素（差分+ZigZag） +// for (int i = 1; i < size; i++) { +// BigDecimal current = scaledValues[i].subtract(min); +// result[i]=current.toBigInteger().intValue(); +// } +// +// return result; +// } + public static int[] sprintz(int[] numbers) { + int size = numbers.length; + int[] result = new int[size]; + + int first = numbers[0]; + result[0] = first; + + // 3. Process subsequent elements with delta + ZigZag encoding + int prev = first; + for (int i = 1; i < size; i++) { + int current = numbers[i]; + int diff = current - prev; + result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding + prev = current; + } + + return result; + } + + + /** + * Sprintz解码 - 从差分编码恢复原始数据 + */ + public static int[] sprintzDecode(int[] encodedData) { + int size = encodedData.length; + int[] result = new int[size]; + + if (size == 0) return result; + + // 第一个元素是原始值 + result[0] = encodedData[0]; + + // 后续元素需要ZigZag解码和累加 + int prev = result[0]; + for (int i = 1; i < size; i++) { + int zigzagEncoded = encodedData[i]; + int diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 + result[i] = prev + diff; + prev = result[i]; + } + + return result; + } + + public static int[] decodeRLE(byte[] data, int startPos, int runCount) { + List bitWidths = new ArrayList<>(); + + for (int i = 0; i < runCount; i++) { + int runLength = data[startPos + i * 2] & 0xFF; + int value = data[startPos + i * 2 + 1] & 0xFF; + + // 重复添加runLength次value + for (int j = 0; j < runLength; j++) { + bitWidths.add(value); + } + } + + // 转换为数组 + int[] result = new int[bitWidths.size()]; + for (int i = 0; i < bitWidths.size(); i++) { + result[i] = bitWidths.get(i); + } + return result; + } + // 新增的解压函数 + public static int[] decodeBitPackingWithRLE(byte[] compressedData, int originalLength, int pack_size) { + List result = new ArrayList<>(); + int pos = 0; + + // 1. 解析RLE编码的bitWidths + // 读取run_count（4字节） + int runCount = bytes2Integer(compressedData, pos, 4); + pos += 4; + + // 解析RLE游程 + int[] bitWidths = decodeRLE(compressedData, pos, runCount); + pos += runCount * 2; // 每个游程占2字节 + + // 2. 解压bit-packed数据 + int totalGroups = bitWidths.length; + + for (int group = 0; group < totalGroups; group++) { + int bitWidth = bitWidths[group]; + + // 解压当前分组 + ArrayList groupData = new ArrayList<>(); + unpack8Values(compressedData, pos, bitWidth, groupData); + + // 添加解压出的数据 + for (int i = 0; i < pack_size; i++) { + if (result.size() < originalLength) { + result.add(groupData.get(i)); + } + } + + pos += bitWidth; + } + + // 转换为数组返回 + int[] decodedArray = new int[result.size()]; + for (int i = 0; i < result.size(); i++) { + decodedArray[i] = result.get(i); + } + return decodedArray; + } + + /** + * 实际的压缩编码函数：将paddedArray按照bitWidths进行bit-packing，并对bitWidths进行RLE编码 + */ + public static byte[] encodeBitPackingWithRLE(int[] paddedArray, int[] bitWidths, int pack_size, int cost_bits) { + List result = new ArrayList<>(); + + // 1. 对bitWidths进行RLE编码 + List rleEncoded = encodeRLE(bitWidths); + + // 2. 将RLE编码的bitWidths写入结果 + // 首先写入RLE数据的长度（4字节） +// int rleLength = rleEncoded.size(); +// result.add((byte) (rleLength >> 24)); +// result.add((byte) (rleLength >> 16)); +// result.add((byte) (rleLength >> 8)); +// result.add((byte) rleLength); + + // 写入RLE数据 + result.addAll(rleEncoded); + + // 3. 对paddedArray进行bit-packing + int totalGroups = bitWidths.length; + + // 计算bit-packed数据的总字节数 - 修正计算方式 + int totalBitPackedBytes = (cost_bits+7)/8; +// for (int i = 0; i < totalGroups; i++) { +// // 每组需要 ceil(8 * bitWidth / 8) = bitWidth 字节 +// totalBitPackedBytes += bitWidths[i]; +// } + + // 确保数组足够大，添加一些额外空间以防万一 + byte[] bitPackedData = new byte[totalBitPackedBytes + 32]; + int encodePos = 0; +// System.out.println(Arrays.toString(bitWidths)); +// System.out.println(result.size()); + // 对每组数据进行bit-packing + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + ArrayList groupData = new ArrayList<>(); + for (int i = 0; i < pack_size; i++) { + if (startIndex + i < paddedArray.length) { + groupData.add(paddedArray[startIndex + i]); + } else { + groupData.add(0); // 用0填充不足的部分 + } + } + + // 确保不会越界 +// if (encodePos + bitWidths[group] <= bitPackedData.length) { + encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); +// } else { +// // 如果空间不足，扩展数组 +// byte[] newBitPackedData = new byte[bitPackedData.length + 32]; +// System.arraycopy(bitPackedData, 0, newBitPackedData, 0, bitPackedData.length); +// bitPackedData = newBitPackedData; +// encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); +// } + } + + // 4. 将bit-packed数据写入结果（只写入实际使用的部分） + for (int i = 0; i < encodePos; i++) { + result.add(bitPackedData[i]); + } + + // 转换为byte数组返回 + byte[] finalResult = new byte[result.size()]; + for (int i = 0; i < result.size(); i++) { + finalResult[i] = result.get(i); + } + + return finalResult; + } + + /** + * RLE编码bitWidths数组 + * chunksize = 1024 + * packsize = 8 + * runlength = 128 + * runcount = + */ + public static List encodeRLE(int[] bitWidths) { + List result = new ArrayList<>(); + + if (bitWidths.length == 0) { + return result; + } + int length_bitWidths_list = bitWidths.length; +// int currentValue = bitWidths[0]; +// int runLength = 1; + int run_count = 0; + + int[] run_lengths = new int[length_bitWidths_list]; + int[] run_values = new int[length_bitWidths_list]; + int pre_bit_width = bitWidths[0]; + int pre_run_length = 1; + + for (int i = 1; i < length_bitWidths_list; i++) { + if (bitWidths[i] == pre_bit_width) { + pre_run_length++; + } else { + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + // 写入当前游程 +// encodeRLERun(result, runLength, currentValue); + pre_bit_width = bitWidths[i]; + pre_run_length = 1; + } + } + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + + result.add((byte) (run_count >> 24)); + result.add((byte) (run_count >> 16)); + result.add((byte) (run_count >> 8)); + result.add((byte) run_count); + for (int i = 0; i < run_count; i++) { + encodeRLERun(result, run_lengths[i], run_values[i]); + } + // 写入最后一个游程 +// encodeRLERun(result, runLength, currentValue); + + return result; + } + + /** + * 编码单个RLE游程 + */ + private static void encodeRLERun(List result, int runLength, int value) { + // 使用变长编码存储游程长度 +// while (runLength > 0) { +// int byteValue = runLength & 0xFF; // 取7位 +// runLength >>= 7; +// if (runLength > 0) { +// byteValue |= 0x80; // 设置最高位表示还有后续字节 +// } +// result.add((byte) (runLength >> 24)); +// result.add((byte) (runLength >> 16)); +// result.add((byte) (runLength >> 8)); + result.add((byte) runLength); +// result.add((byte) (value >> 24)); +// result.add((byte) (value >> 16)); +// result.add((byte) (value >> 8)); + result.add((byte) value); + + } + + public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { + int blocksize= bitWidths.length; + + int totalCost = 0; + int numBlocks = (int) Math.ceil((double) blocksize / fixed_pack); + // RLE compress bit width series: rle_count 8 bits, (run length 8 bits, bit width value: 6 bits) * run_count + + // Calculate cost for each pack + for (int pack = 0; pack < numBlocks; pack++) { + int start = pack * fixed_pack; + int end = Math.min(start + fixed_pack, blocksize); + int cur_block_size = end - start; + + // Find max bitWidth in current pack + int maxBitWidth = 0; + int[] run_lengths = new int[cur_block_size]; + int[] run_values = new int[cur_block_size]; + int run_count = 0; + int pre_bit_width = bitWidths[start]; + int pre_run_length = 1; + totalCost += pack_size * bitWidths[start]; + + for (int i = start+1; i < end; i++) { + totalCost += pack_size * bitWidths[i]; + if(pre_bit_width == bitWidths[i]) { + pre_run_length++; + } else { + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + pre_bit_width = bitWidths[i]; + pre_run_length = 1; + } + } + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + +// System.out.println(run_count); + totalCost += 64; + for (int i = 0; i < run_count; i++) { + totalCost += 32; + } + + + } + System.out.println(blocksize); + // Store max bit width +// totalCost += 5 * blocksize / fixed_pack; + + + return totalCost; + } + + public static void main(String[] args) throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_sprintz"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / 8] = bitWidth; + cost_bits += (bitWidth*8); + } + + + // 替换computeMinPackingCost为实际的压缩编码 + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, 8,cost_bits); + int cur_cost = compressedData.length * 8; // 转换为bit数 +// System.out.println(cur_cost); + long duration = System.nanoTime() - startTime; + + long decodeStartTime = System.nanoTime(); + int[] decompressedPacked = decodeBitPackingWithRLE(compressedData, scaledInts.length, 8); + int[] decompressedScaled = sprintzDecode(decompressedPacked); + long decodeDuration = System.nanoTime() - decodeStartTime; + + modelTime += duration; + modelDecodeTime += decodeDuration; + modelCost += cur_cost; + } + } + modelCost /= time_of_repeat; + modelTime = modelTime / time_of_repeat; + modelDecodeTime = modelDecodeTime / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000) / (double) (modelTime); + double decodeThroughput = (double) (numbers.size() * 8000) / (double) modelDecodeTime; + String[] record = { + file.toString(), + "BP+RLE-Sprintz", + String.valueOf(modelTime_throughput), + String.valueOf(decodeThroughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); +// break; + } + } + + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_SPRINTZ_RLE_vary_pack_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++){ + int pack_size = (int) Math.pow(2,pack_size_exp); + int modelCost = 0; + long modelTime = 0; + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + cost_bits += (bitWidth*pack_size); + } + + int fixed_block = CHUNK_SIZE / pack_size; + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); + int cur_cost = compressedData.length * 8; // 转换为bit数 + + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /=time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); + String[] record = { + file.toString(), + "SPRINTZ+RLE", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + + writer.close(); + } + + } + // 新增方法：测试不同chunk size的表现 + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_RLE_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); +// +// int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; + cost_bits += (bitWidth * pack_size); + } + + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = modelTime / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "Sprintz+RLE", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); + } + } + +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java new file mode 100644 index 000000000..60bf431ac --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java @@ -0,0 +1,858 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RLEPackBitWidthTest { + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", + "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); + private static final int CHUNK_SIZE = 1024; + + public static int getBitWith(int num) { + if (num == 0) + return 1; + else + return 32 - Integer.numberOfLeadingZeros(num); + } + + public static int getCount(long long1, int mask) { + return ((int) (long1 & mask)); + } + + public static int getUniqueValue(long long1, int left_shift) { + return ((int) ((long1) >> left_shift)); + } + + public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer); + } + + private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + if (num > 4) { + System.out.println("bytes2Integer error"); + return 0; + } + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + private static long bytesLong2Integer(byte[] encoded, int decode_pos) { + long value = 0; + for (int i = 0; i < 4; i++) { + value <<= 8; + int b = encoded[i + decode_pos] & 0xFF; + value |= b; + } + return value; + } + + public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + // remaining bits for the current unfinished Integer + int leftBit = 0; + + while (valueIdx < 8 + offset) { + // buffer is used for saving 32 bits as a part of result + int buffer = 0; + // remaining size of bits in the 'buffer' + int leftSize = 32; + + // encode the left bits of current Integer to 'buffer' + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + // encode one Integer to the 'buffer' + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + // If the remaining space of the buffer can not save the bits for one Integer, + if (leftSize > 0 && valueIdx < 8 + offset) { + // put the first 'leftSize' bits of the Integer into remaining space of the + // buffer + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + // put the buffer into the final result + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { + int byteIdx = offset; + long buffer = 0; + // total bits which have read from 'buf' to 'buffer'. i.e., + // number of available bits to be decoded. + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + // If current available bits are not enough to decode one Integer, + // then add next byte from buf to 'buffer' until totalBits >= width + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + // If current available bits are enough to decode one Integer, + // then decode one Integer one by one until left bits in 'buffer' is + // not enough to decode one Integer. + while (totalBits >= width && valueIdx < 8) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, + byte[] encoded_result) { + int block_num = (numbers.size() - start) / 8; + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + return encode_pos; + + } + + public static ArrayList decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int block_size) { + ArrayList result_list = new ArrayList<>(); + int block_num = (block_size - 1) / 8; + + for (int i = 0; i < block_num; i++) { // bitpacking + unpack8Values(encoded, decode_pos, bit_width, result_list); + decode_pos += bit_width; + } + return result_list; + } + + // 新增的解压函数 + public static int[] decodeBitPackingWithRLE(byte[] compressedData, int originalLength, int pack_size) { + List result = new ArrayList<>(); + int pos = 0; + + // 1. 解析RLE编码的bitWidths + // 读取run_count（4字节） + int runCount = bytes2Integer(compressedData, pos, 4); + pos += 4; + + // 解析RLE游程 + int[] bitWidths = decodeRLE(compressedData, pos, runCount); + pos += runCount * 2; // 每个游程占2字节 + + // 2. 解压bit-packed数据 + int totalGroups = bitWidths.length; + + for (int group = 0; group < totalGroups; group++) { + int bitWidth = bitWidths[group]; + + // 解压当前分组 + ArrayList groupData = new ArrayList<>(); + unpack8Values(compressedData, pos, bitWidth, groupData); + + // 添加解压出的数据 + for (int i = 0; i < pack_size; i++) { + if (result.size() < originalLength) { + result.add(groupData.get(i)); + } + } + + pos += bitWidth; + } + + // 转换为数组返回 + int[] decodedArray = new int[result.size()]; + for (int i = 0; i < result.size(); i++) { + decodedArray[i] = result.get(i); + } + return decodedArray; + } + + // 新增的RLE解码函数 + public static int[] decodeRLE(byte[] data, int startPos, int runCount) { + List bitWidths = new ArrayList<>(); + + for (int i = 0; i < runCount; i++) { + int runLength = data[startPos + i * 2] & 0xFF; + int value = data[startPos + i * 2 + 1] & 0xFF; + + // 重复添加runLength次value + for (int j = 0; j < runLength; j++) { + bitWidths.add(value); + } + } + + // 转换为数组 + int[] result = new int[bitWidths.size()]; + for (int i = 0; i < bitWidths.size(); i++) { + result[i] = bitWidths.get(i); + } + return result; + } + + private static int[] scaleNumbers(List numbers, int decimalMax) { + // 1. 预先计算缩放因子 + BigDecimal scale = BigDecimal.TEN.pow(decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + + if (size == 0) { + return result; + } + + // 2. 单次遍历完成所有转换和最小值查找 + BigDecimal min = null; + BigDecimal[] scaledValues = new BigDecimal[size]; + + for (int i = 0; i < size; i++) { + BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); + scaledValues[i] = val; + if (min == null || val.compareTo(min) < 0) { + min = val; + } + } + + // 3. 处理第一个元素 + BigDecimal first = scaledValues[0].subtract(min); + result[0] = first.toBigInteger().intValue(); + + // 4. 处理后续元素（差分+ZigZag） + for (int i = 1; i < size; i++) { + BigDecimal current = scaledValues[i].subtract(min); + result[i]=current.toBigInteger().intValue(); + } + + return result; + } + + /** + * 实际的压缩编码函数：将paddedArray按照bitWidths进行bit-packing，并对bitWidths进行RLE编码 + */ + public static byte[] encodeBitPackingWithRLE(int[] paddedArray, int[] bitWidths, int pack_size, int cost_bits) { + List result = new ArrayList<>(); + + // 1. 对bitWidths进行RLE编码 + List rleEncoded = encodeRLE(bitWidths); + + // 2. 将RLE编码的bitWidths写入结果 + // 写入RLE数据 + result.addAll(rleEncoded); + + // 3. 对paddedArray进行bit-packing + int totalGroups = bitWidths.length; + + // 计算bit-packed数据的总字节数 - 修正计算方式 + int totalBitPackedBytes = (cost_bits+7)/8; + + // 确保数组足够大，添加一些额外空间以防万一 + byte[] bitPackedData = new byte[totalBitPackedBytes + 32]; + int encodePos = 0; + + // 对每组数据进行bit-packing + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + ArrayList groupData = new ArrayList<>(); + for (int i = 0; i < pack_size; i++) { + if (startIndex + i < paddedArray.length) { + groupData.add(paddedArray[startIndex + i]); + } else { + groupData.add(0); // 用0填充不足的部分 + } + } + + encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); + } + + // 4. 将bit-packed数据写入结果（只写入实际使用的部分） + for (int i = 0; i < encodePos; i++) { + result.add(bitPackedData[i]); + } + + // 转换为byte数组返回 + byte[] finalResult = new byte[result.size()]; + for (int i = 0; i < result.size(); i++) { + finalResult[i] = result.get(i); + } + + return finalResult; + } + + /** + * RLE编码bitWidths数组 + * chunksize = 1024 + * packsize = 8 + * runlength = 128 + * runcount = + */ + public static List encodeRLE(int[] bitWidths) { + List result = new ArrayList<>(); + + if (bitWidths.length == 0) { + return result; + } + int length_bitWidths_list = bitWidths.length; + int run_count = 0; + + int[] run_lengths = new int[length_bitWidths_list]; + int[] run_values = new int[length_bitWidths_list]; + int pre_bit_width = bitWidths[0]; + int pre_run_length = 1; + + for (int i = 1; i < length_bitWidths_list; i++) { + if (bitWidths[i] == pre_bit_width) { + pre_run_length++; + } else { + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + pre_bit_width = bitWidths[i]; + pre_run_length = 1; + } + } + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + + result.add((byte) (run_count >> 24)); + result.add((byte) (run_count >> 16)); + result.add((byte) (run_count >> 8)); + result.add((byte) run_count); + for (int i = 0; i < run_count; i++) { + encodeRLERun(result, run_lengths[i], run_values[i]); + } + + return result; + } + + /** + * 编码单个RLE游程 + */ + private static void encodeRLERun(List result, int runLength, int value) { + result.add((byte) (runLength >> 24)); + result.add((byte) (runLength >> 16)); + result.add((byte) (runLength >> 8)); + result.add((byte) runLength); + result.add((byte) (runLength >> 24)); + result.add((byte) (value >> 16)); + result.add((byte) (value >> 8)); + result.add((byte) value); + } + + public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { + int blocksize= bitWidths.length; + + int totalCost = 0; + int numBlocks = (int) Math.ceil((double) blocksize / fixed_pack); + // RLE compress bit width series: rle_count 8 bits, (run length 8 bits, bit width value: 6 bits) * run_count + + // Calculate cost for each pack + for (int pack = 0; pack < numBlocks; pack++) { + int start = pack * fixed_pack; + int end = Math.min(start + fixed_pack, blocksize); + int cur_block_size = end - start; + + // Find max bitWidth in current pack + int maxBitWidth = 0; + int[] run_lengths = new int[cur_block_size]; + int[] run_values = new int[cur_block_size]; + int run_count = 0; + int pre_bit_width = bitWidths[start]; + int pre_run_length = 1; + totalCost += pack_size * bitWidths[start]; + + for (int i = start+1; i < end; i++) { + totalCost += pack_size * bitWidths[i]; + if(pre_bit_width == bitWidths[i]) { + pre_run_length++; + } else { + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + pre_bit_width = bitWidths[i]; + pre_run_length = 1; + } + } + run_lengths[run_count] = pre_run_length; + run_values[run_count++] = pre_bit_width; + + totalCost += 64; + for (int i = 0; i < run_count; i++) { + totalCost += 32; + } + } + System.out.println(blocksize); + + return totalCost; + } + + public static void main(String[] args) throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + // 更新表头，增加解压吞吐率列 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; // 新增：解压时间统计 + + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % 8; + int paddingLength = (remainder == 0) ? 0 : 8 - remainder; + + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / 8]; + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / 8] = bitWidth; + cost_bits += (bitWidth*8); + } + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, 8,cost_bits); + long cur_cost = compressedData.length * 8; // 转换为bit数 + + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += cur_cost; + + // 新增：测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingWithRLE(compressedData, scaledInts.length, 8); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + +// // 可选：验证解压数据的正确性（只在第一次重复时验证） +// if (j == 0) { +// boolean correct = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// correct = false; +// System.err.println("Decompression error at index " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// break; +// } +// } +//// if (correct) { +//// System.out.println("Decompression verified successfully for chunk " + (i/CHUNK_SIZE)); +//// } +// } + } + } + modelCost /= time_of_repeat; + modelTime = modelTime / time_of_repeat; + modelDecodeTime = modelDecodeTime / time_of_repeat; // 平均解压时间 + + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000L) / (double) (modelTime); // points per second + double modelDecodeTime_throughput = (double)(numbers.size()*8000L) / (double) (modelDecodeTime); // points per second + + // 更新输出记录，包含解压吞吐率 + String[] record = { + file.toString(), + "BP+RLE", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + +// System.out.println("Encoding throughput: " + modelTime_throughput + " points/s"); +// System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " points/s"); +// System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void TestVarPackSize() throws IOException { + // 示例数据（实际应替换为真实时间序列） + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_vary_pack_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); // write header to output file + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++){ + int pack_size = (int) Math.pow(2,pack_size_exp); + System.out.println(pack_size); + int modelCost = 0; + long modelTime = 0; + for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); + if(chunkNumbers.size()==1 || chunkNumbers.size()==2) + continue; + + int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) + .stream().max(Integer::compare).orElse(0); + + int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为8的倍数 + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + // 1. 找出当前8个元素中的最大值 + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + + // 2. 计算该最大值的去头零位宽 + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + + // 3. 存储结果 + bitWidths[scaledInts_i / pack_size] = bitWidth; + cost_bits += (bitWidth*pack_size); + } + + int fixed_block = CHUNK_SIZE / pack_size; + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); + int cur_cost = compressedData.length * 8; // 转换为bit数 + + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + } + + } + modelCost /=time_of_repeat; + modelTime = (modelTime)/time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size()*64); + double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); + String[] record = { + file.toString(), + "BP+RLE", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + + writer.close(); + } + + } + + // 新增方法：测试不同chunk size的表现 + @Test + public void TestVariableChunkSize() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_vary_m"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Points", + "Compressed Size", + "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + long modelTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); +// +// int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + int remainder = scaledInts.length % pack_size; + int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + int[] paddedArray = new int[scaledInts.length + paddingLength]; + System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); + int actual_length = paddedArray.length; + int[] bitWidths = new int[actual_length / pack_size]; + + int cost_bits = 0; + for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { + int maxInGroup = 0; + for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { + if (paddedArray[scaledInts_j] > maxInGroup) { + maxInGroup = paddedArray[scaledInts_j]; + } + } + int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); + bitWidths[scaledInts_i / pack_size] = bitWidth; + cost_bits += (bitWidth * pack_size); + } + + byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + modelTime += duration; + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = modelTime / time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + + String[] record = { + String.valueOf(chunkSize/8), + file.toString(), + "BP+RLE", + String.valueOf(modelTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); + } + } +} \ No newline at end of file From e8a25d191b682df6155bb364042495096a164f55 Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Fri, 2 Jan 2026 08:42:15 +0800 Subject: [PATCH 2/9] update --- .../encoding/AllNo8PacksizeOptimal.java | 7498 +++++++++++++++++ .../org/apache/tsfile/encoding/BDCTest.java | 575 -- .../tsfile/encoding/DPOctadPacking.java | 843 -- .../encoding/DPOctadPackingSprintz.java | 941 --- .../encoding/EfficientOctadPackingMLP.java | 1321 --- .../EfficientOctadPackingMLPSprintz.java | 1504 ---- .../tsfile/encoding/FBitpacking512.java | 798 -- .../apache/tsfile/encoding/FSprintz512.java | 1158 --- .../encoding/RLEPackBitWidthSprintzTest.java | 916 -- .../tsfile/encoding/RLEPackBitWidthTest.java | 858 -- 10 files changed, 7498 insertions(+), 8914 deletions(-) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java new file mode 100644 index 000000000..ded951ca9 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -0,0 +1,7498 @@ +package org.apache.iotdb.tsfile.encoding; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; + +import static org.apache.commons.lang3.ObjectUtils.min; + +public class AllNo8PacksizeOptimal { + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv", "POI-lat.csv", + "POI-lon.csv", "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv"); //,"Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv" + private static final int CHUNK_SIZE = 1024; + + public static int getCount(long long1, int mask) { + return ((int) (long1 & mask)); + } + + /** + * 修改后的编码函数，支持任意大小的pack_size + * 返回实际写入的字节数 + */ + public static int packValues(ArrayList values, int offset, int count, int width, int encode_pos, + byte[] encoded_result) { + if (count <= 0) return encode_pos; + + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + int totalBits = count * width; + int totalBytes = (totalBits + 7) / 8; // 向上取整到字节 + + while (valueIdx < count + offset) { + int buffer = 0; + int leftSize = 32; + + if (leftBit > 0) { + buffer |= (values.get(valueIdx) << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < count + offset) { + buffer |= (values.get(valueIdx) << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + + if (leftSize > 0 && valueIdx < count + offset) { + buffer |= (values.get(valueIdx) >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 4 && bufIdx < totalBytes; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= totalBytes) { + return encode_pos; + } + } + } + return encode_pos; + } + + /** + * 修改后的解码函数，支持任意数量的值 + */ + public static void unpackValues(byte[] encoded, int offset, int count, int width, ArrayList result_list) { + if (count <= 0) return; + + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + int totalBitsNeeded = count * width; + int totalBytes = (totalBitsNeeded + 7) / 8; + + while (valueIdx < count) { + while (totalBits < width && byteIdx - offset < totalBytes) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < count) { + result_list.add((int) (buffer >>> (totalBits - width))); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + /** + * 通用的bitpacking编码函数，支持任意pack_size + */ + public static int bitPacking(ArrayList numbers, int start, int count, int bit_width, int encode_pos, + byte[] encoded_result) { + if (count <= 0) return encode_pos; + + // 计算需要的字节数 + int totalBits = count * bit_width; + int bytesNeeded = (totalBits + 7) / 8; + + // 确保数组有足够空间 + if (encode_pos + bytesNeeded > encoded_result.length) { + byte[] newArray = new byte[encode_pos + bytesNeeded]; + System.arraycopy(encoded_result, 0, newArray, 0, encode_pos); + encoded_result = newArray; + } + + return packValues(numbers, start, count, bit_width, encode_pos, encoded_result); + } + + /** + * 解码函数 - 支持可变大小的数据块和非8倍数的pack_size + */ + public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { + int[] result = new int[originalLength]; + int resultIndex = 0; + + // 1. 从压缩数据中解析 bitWidths + int totalGroups = bitWidths.length; + int bitWidthBits = totalGroups * 6; + int bitWidthBytes = (bitWidthBits + 7) / 8; // 向上取整到字节 + + int[] decodedBitWidths = new int[totalGroups]; + + // 从压缩数据中读取 bitWidths + int bitPos = 0; + for (int group = 0; group < totalGroups; group++) { + int bitWidth = 0; + + // 读取 6 位 + for (int bit = 0; bit < 6; bit++) { + int byteIndex = bitPos / 8; + int bitOffset = 7 - (bitPos % 8); // 高位在前 + + int bitValue = (compressedData[byteIndex] >> bitOffset) & 1; + bitWidth = (bitWidth << 1) | bitValue; + + bitPos++; + } + + decodedBitWidths[group] = bitWidth; + } + + // 2. 解码数据部分 + int decodePos = bitWidthBytes; + + for (int group = 0; group < totalGroups && resultIndex < originalLength; group++) { + int bitWidth = decodedBitWidths[group]; + + // 计算这个数据块中需要解码的值数量 + int valuesInGroup = Math.min(pack_size, originalLength - resultIndex); + if (valuesInGroup <= 0) break; + + // 计算这个数据块需要的字节数 + int bitsNeeded = valuesInGroup * bitWidth; + int bytesNeeded = (bitsNeeded + 7) / 8; + + // 解码这个数据块 + ArrayList groupData = new ArrayList<>(valuesInGroup); + unpackValues(compressedData, decodePos, valuesInGroup, bitWidth, groupData); + + // 将解码的值复制到结果数组中 + for (int i = 0; i < groupData.size() && resultIndex < originalLength; i++) { + result[resultIndex++] = groupData.get(i); + } + + // 移动到下一个数据块 + decodePos += bytesNeeded; + } + + // 3. 将解码出的 bitWidths 复制回传入的数组 + System.arraycopy(decodedBitWidths, 0, bitWidths, 0, totalGroups); + + return result; + } +// public static int[] decodeBitPackingV2(byte[] encodedData,int[] originalBitWidths, int pack_size, int n) { +// if (originalBitWidths == null || originalBitWidths.length == 0) { +// throw new IllegalArgumentException("bitWidths array is required for decoding"); +// } +// +// int totalGroups = originalBitWidths.length; +// int[] decodedArray = new int[n]; +// +// // 1. 计算bitWidths数组的最大值，确定存储位数 +// int maxBitWidth = 0; +// for (int bitWidth : originalBitWidths) { +// if (bitWidth > maxBitWidth) { +// maxBitWidth = bitWidth; +// } +// } +// int bitsForBitWidth = 32 - Integer.numberOfLeadingZeros(maxBitWidth); +// +// // 2. 创建BitReader来读取bits +// BitReader bitReader = new BitReader(encodedData); +// +// // 3. 从编码数据中读取bitWidths（验证用） +// int[] decodedBitWidths = new int[totalGroups]; +// for (int group = 0; group < totalGroups; group++) { +// decodedBitWidths[group] = bitReader.readBits(bitsForBitWidth); +// +// // 验证bitWidth是否匹配（可选） +// if (decodedBitWidths[group] != originalBitWidths[group]) { +// throw new IllegalStateException("BitWidth mismatch at group " + group + +// ": expected " + originalBitWidths[group] + +// ", got " + decodedBitWidths[group]); +// } +// } +// +// // 4. 解码数据部分 +// for (int group = 0; group < totalGroups; group++) { +// int startIndex = group * pack_size; +// int bitWidth = originalBitWidths[group]; +// +// int valuesInGroup = Math.min(pack_size, n - startIndex); +// if (valuesInGroup <= 0) break; +// +// for (int i = 0; i < valuesInGroup; i++) { +// int idx = startIndex + i; +// if (idx < n) { +// decodedArray[idx] = bitReader.readBits(bitWidth); +// } else { +// // 跳过多余的值 +// bitReader.readBits(bitWidth); +// } +// } +// } +// +// return decodedArray; +// } + + public static byte[] encodeBitPackingV2(int[] originalArray, int[] bitWidths, int pack_size) { + int totalGroups = bitWidths.length; + int n = originalArray.length; + + // 1. 计算存储bitWidths所需的位数（根据最大值动态确定） + int maxBitWidth = 0; + for (int bitWidth : bitWidths) { + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 计算存储每个bitWidth需要的位数 + int bitsForBitWidth; + if (maxBitWidth == 0) { + bitsForBitWidth = 1; // 至少1位 + } else { + bitsForBitWidth = 32 - Integer.numberOfLeadingZeros(maxBitWidth); + } + + + // 4. 使用BitWriter来按bit写入 + BitWriterV2 bitWriter = new BitWriterV2(); + + // 5. 编码bitWidths信息（用动态位数存储） + // 首先写入bitsForBitWidth本身（用固定6位存储，因为bitWidth最大值不会超过32） + bitWriter.writeBits(bitsForBitWidth, 6); + + // 然后写入每个group的bitWidth + for (int bitWidth : bitWidths) { + bitWriter.writeBits(bitWidth, bitsForBitWidth); + } + + // 6. 编码数据部分（按bit连续存储） + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + int bitWidth = bitWidths[group]; + + int valuesInGroup = Math.min(pack_size, n - startIndex); + if (valuesInGroup <= 0) break; + + // 编码这个pack的所有值 + for (int i = 0; i < valuesInGroup; i++) { + int idx = startIndex + i; +// if (idx < n) { + int value = originalArray[idx]; + // 确保value在bitWidth范围内 + int maskedValue = value & ((1 << bitWidth) - 1); + bitWriter.writeBits(maskedValue, bitWidth); +// } else { +// // 填充0 +// bitWriter.writeBits(0, bitWidth); +// } + } + } + + + + return bitWriter.toByteArray(); + } + + // 对应的解码函数 + public static int[] decodeBitPackingV2(byte[] encodedData, int[] originalBitWidths , int pack_size, int n) { + if (originalBitWidths == null || originalBitWidths.length == 0) { + throw new IllegalArgumentException("bitWidths array is required for decoding"); + } + + int totalGroups = originalBitWidths.length; + int[] decodedArray = new int[n]; + + // 1. 创建BitReader + BitReaderV2 bitReader = new BitReaderV2(encodedData); + + // 2. 首先读取bitsForBitWidth（固定6位） + int bitsForBitWidth = bitReader.readBits(6); + + // 3. 从编码数据中读取bitWidths（用于验证） + int[] decodedBitWidths = new int[totalGroups]; + for (int group = 0; group < totalGroups; group++) { + decodedBitWidths[group] = bitReader.readBits(bitsForBitWidth); + +// // 验证bitWidth是否匹配 +// if (decodedBitWidths[group] != originalBitWidths[group]) { +// System.err.println("BitWidth mismatch at group " + group + +// ": expected " + originalBitWidths[group] + +// ", got " + decodedBitWidths[group]); +// // 这里可以选择使用原始值继续解码 +// // decodedBitWidths[group] = originalBitWidths[group]; +// } + } + + // 4. 解码数据部分 + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + int bitWidth = originalBitWidths[group]; // 使用原始的bitWidth + + int valuesInGroup = Math.min(pack_size, n - startIndex); + if (valuesInGroup <= 0) break; + + for (int i = 0; i < valuesInGroup; i++) { + int idx = startIndex + i; +// if (idx < n) { + decodedArray[idx] = bitReader.readBits(bitWidth); +// } else { +// // 跳过多余的值 +// bitReader.readBits(bitWidth); +// } + } + } + + return decodedArray; + } + + // 改进的BitWriterV2类 + static class BitWriterV2 { + private ByteArrayOutputStream buffer; + private int currentByte; + private int bitPosition; // 0-7，当前字节中的bit位置 + + public BitWriterV2() { + this.buffer = new ByteArrayOutputStream(); + this.currentByte = 0; + this.bitPosition = 7; // 从最高位开始 + } + + // 写入指定数量的bits（从value的低位开始） + public void writeBits(int value, int numBits) { + if (numBits <= 0) return; + + for (int i = numBits - 1; i >= 0; i--) { + int bit = (value >> i) & 1; + + // 设置当前bit + if (bit == 1) { + currentByte |= (1 << bitPosition); + } + + // 移动到下一个bit位置 + bitPosition--; + if (bitPosition < 0) { + // 保存当前字节并重置 + buffer.write(currentByte); + currentByte = 0; + bitPosition = 7; + } + } + } + + // 获取编码结果 + public byte[] toByteArray() { + // 如果还有未写入的bit，写入最后一个字节 + if (bitPosition != 7) { + buffer.write(currentByte); + } + return buffer.toByteArray(); + } + + // 获取已写入的bit数 + public int getBitsWritten() { + int bytesWritten = buffer.size(); + if (bitPosition != 7) { + bytesWritten++; // 包括当前字节 + } + return bytesWritten * 8 - (bitPosition + 1); + } + } + + // 改进的BitReaderV2类 + static class BitReaderV2 { + private byte[] buffer; + private int currentByteIndex; + private int currentByte; + private int bitPosition; // 0-7，当前字节中的bit位置 + + public BitReaderV2(byte[] buffer) { + this.buffer = buffer; + this.currentByteIndex = 0; + if (buffer.length > 0) { + this.currentByte = buffer[0] & 0xFF; + } else { + this.currentByte = 0; + } + this.bitPosition = 7; // 从最高位开始 + } + + // 读取指定数量的bits + public int readBits(int numBits) { + if (numBits <= 0) return 0; + + int result = 0; + for (int i = 0; i < numBits; i++) { + // 读取当前bit + int bit = (currentByte >> bitPosition) & 1; + result = (result << 1) | bit; + + // 移动到下一个bit位置 + bitPosition--; + if (bitPosition < 0) { + // 移动到下一个字节 + currentByteIndex++; + if (currentByteIndex < buffer.length) { + currentByte = buffer[currentByteIndex] & 0xFF; + } else { + currentByte = 0; // 超出范围返回0 + } + bitPosition = 7; + } + } + + return result; + } + + // 跳过指定数量的bits + public void skipBits(int numBits) { + for (int i = 0; i < numBits; i++) { + bitPosition--; + if (bitPosition < 0) { + currentByteIndex++; + if (currentByteIndex < buffer.length) { + currentByte = buffer[currentByteIndex] & 0xFF; + } else { + currentByte = 0; + } + bitPosition = 7; + } + } + } + + // 获取当前位置（bit数） + public int getPosition() { + return currentByteIndex * 8 + (7 - bitPosition); + } + } + // BitReader辅助类，用于按bit读取 + static class BitReader { + private byte[] buffer; + private int currentByte; + private int bitPosition; // 0-7，当前字节中的bit位置 + + public BitReader(byte[] buffer) { + this.buffer = buffer; + this.currentByte = 0; + this.bitPosition = 7; // 从最高位开始 + } + + // 读取指定数量的bits + public int readBits(int numBits) { + if (numBits <= 0) return 0; + + int result = 0; + for (int i = 0; i < numBits; i++) { + // 读取当前bit + int bit = (buffer[currentByte] >> bitPosition) & 1; + result = (result << 1) | bit; + + // 移动到下一个bit位置 + bitPosition--; + if (bitPosition < 0) { + // 移动到下一个字节 + currentByte++; + bitPosition = 7; + } + } + + return result; + } + + // 跳过一个bit + public void skipBit() { + bitPosition--; + if (bitPosition < 0) { + currentByte++; + bitPosition = 7; + } + } + + // 跳过多个bits + public void skipBits(int numBits) { + for (int i = 0; i < numBits; i++) { + skipBit(); + } + } + } + + /** + * 使用RMQ（稀疏表）快速计算区间最大值，找到最优的pack_size 返回8的倍数 + */ + public static int findOptimalPackSize(int[] values) { + int n = values.length; + if (n < 8) return n; // 返回实际大小，而不是8 + + // 计算全局最大位宽和z值 + int globalMax = 0; + for (int value : values) { + if (value > globalMax) { + globalMax = value; + } + } + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(globalMax); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 预计算所有值的位宽 + int[] bitWidths = new int[n]; + for (int i = 0; i < n; i++) { + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, values[i])); + } + + // 构建稀疏表用于快速区间最大值查询 + int k = (int)(Math.log(n) / Math.log(2)) + 1; + int[][] st = new int[k][n]; + + // 初始化第一层 + for (int i = 0; i < n; i++) { + st[0][i] = bitWidths[i]; + } + + // 构建稀疏表 + for (int j = 1; j < k; j++) { + for (int i = 0; i + (1 << j) <= n; i++) { + st[j][i] = Math.max(st[j-1][i], st[j-1][i + (1 << (j-1))]); + } + } + + // 区间最大值查询函数 + BiFunction queryMax = (l, r) -> { + int len = r - l + 1; + int j = (int)(Math.log(len) / Math.log(2)); + return Math.max(st[j][l], st[j][r - (1 << j) + 1]); + }; + + // 枚举所有可能的pack_size（现在可以是任意整数，不再限制为8的倍数） + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + + // 限制最大pack_size，避免性能问题 + int maxPackSize = Math.min(CHUNK_SIZE, n); + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + + long cost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + int maxBitWidth = queryMax.apply(start, end); + // 计算实际存储需要的字节数：向上取整到字节 + int totalBits = p * maxBitWidth; + int bytesNeeded = (totalBits + 7) / 8; + cost += bytesNeeded * 8; // 转换为比特 + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int lastMaxBitWidth = queryMax.apply(lastStart, lastEnd); + int totalBits = r * lastMaxBitWidth; + int bytesNeeded = (totalBits + 7) / 8; + cost += bytesNeeded * 8; + } + + // 加上位宽信息的存储成本（每个pack用6位存储位宽） + cost += m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + + return bestPackSize; + } + + + /** + * 暴力计算最优pack_size - O(n^2)复杂度，支持非8倍数 但是非8的字节计算方式要填补 + */ + public static int findOptimalPackSizeall(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算全局最大位宽和z值 + int globalMax = 0; + for (int value : values) { + if (value > globalMax) { + globalMax = value; + } + } + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 枚举所有可能的pack_size + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + + // 限制最大pack_size，避免性能问题 + int maxPackSize = n; + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + + long cost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 计算实际存储需要的字节数 + int totalBits = p * maxBitWidth; + int bytesNeeded = (totalBits + 7) / 8; + cost += bytesNeeded * 8; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + int totalBits = r * lastMaxBitWidth; + int bytesNeeded = (totalBits + 7) / 8; + cost += bytesNeeded * 8; + } + + // 加上位宽信息的存储成本 + cost += (long) m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + + return bestPackSize; + } + + // 遍历所有情况 n^2 + public static int findOptimalPackSizeallV2(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算全局最大位宽和z值（以bit为单位） + int globalMax = 0; + for (int value : values) { + if (value > globalMax) { + globalMax = value; + } + } + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 枚举所有可能的pack_size + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + + // 限制最大pack_size，避免性能问题 + int maxPackSize = n; // 可以调整这个限制 + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + + long cost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + cost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + cost += bitsNeeded; + } + + // 加上位宽信息的存储成本（每个pack需要z位） + cost += (long) m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + +// if(bestPackSize == 1) + + + return bestPackSize; + } + + // 结合RMQ + public static int findOptimalPackSizeallV3(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); // log2(n) + int[][] st = new int[logN][n]; + + // 初始化第一层 + for (int i = 0; i < n; i++) { + st[0][i] = bitWidths[i]; + } + + // 构建稀疏表 + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 枚举所有可能的pack_size + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + int maxPackSize = n; + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + long cost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 使用RMQ查询区间最大值 + int k = log2[p]; + int maxBitWidth = Math.max(st[k][start], st[k][end - (1 << k) + 1]); + + cost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = Math.max(st[k][lastStart], + st[k][lastEnd - (1 << k) + 1]); + cost += (long) r * lastMaxBitWidth; + } + } + + // 加上位宽信息的存储成本 + cost += (long) m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + +// if (bestPackSize == 1) +// System.out.println(bestCost); + + return bestPackSize; + } + // 结合RMQ+剪枝（保证正确性的高效版本） + public static int findOptimalPackSizeallV4(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + int globalMinBitWidth = Integer.MAX_VALUE; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + int width = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + bitWidths[i] = width; + if (width < globalMinBitWidth) { + globalMinBitWidth = width; + } + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + + // 初始化第一层 + for (int i = 0; i < n; i++) { + st[0][i] = bitWidths[i]; + } + + // 构建稀疏表 + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 计算cost函数 + Function computeCost = (Integer p) -> { + int m = (n + p - 1) / p; // ceil(n/p) + long cost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + int k = log2[p]; + int maxBitWidth = Math.max(st[k][start], st[k][end - (1 << k) + 1]); + cost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = Math.max(st[k][lastStart], + st[k][lastEnd - (1 << k) + 1]); + cost += (long) r * lastMaxBitWidth; + } + } + + // 加上位宽信息的存储成本 + cost += (long) m * z; + return cost; + }; + + int maxPackSize = Math.min(CHUNK_SIZE, n); + + // 1. 首先计算p=1的成本作为基准 + long cost1 = computeCost.apply(1); + long bestCost = cost1; + int bestP = 1; + + // 2. 预计算前缀最小位宽（用于计算下界） + int[] prefixMin = new int[n]; + prefixMin[0] = bitWidths[0]; + for (int i = 1; i < n; i++) { + prefixMin[i] = Math.min(prefixMin[i-1], bitWidths[i]); + } + + // 预计算后缀最小位宽 + int[] suffixMin = new int[n]; + suffixMin[n-1] = bitWidths[n-1]; + for (int i = n-2; i >= 0; i--) { + suffixMin[i] = Math.min(suffixMin[i+1], bitWidths[i]); + } + + // 计算下界的函数（使用全局最小值作为近似，计算简单且快速） + int finalGlobalMinBitWidth = globalMinBitWidth; + Function computeLowerBound = (Integer p) -> { + int m = (n + p - 1) / p; + // 使用全局最小值作为每个pack的最小位宽近似，这样得到的下界是安全的（不会过高） + long lowerBound = (long) n * finalGlobalMinBitWidth + (long) m * z; + return lowerBound; + }; + + // 3. 搜索策略：分段搜索 + 剪枝 + + // 分段1：小p值（1-16），直接搜索 + int smallSearchLimit = Math.min(16, maxPackSize); + for (int p = 2; p <= smallSearchLimit; p++) { + long cost = computeCost.apply(p); + if (cost < bestCost) { + bestCost = cost; + bestP = p; + } + } + + // 分段2：中等p值（17-64），使用间隔搜索+剪枝 + if (maxPackSize > 16) { + int mediumStart = 17; + int mediumEnd = Math.min(64, maxPackSize); + + for (int p = mediumStart; p <= mediumEnd; p++) { + // 先计算下界，如果下界已经大于当前最优，则跳过 + long lowerBound = computeLowerBound.apply(p); + if (lowerBound >= bestCost) { + continue; // 可以安全跳过 + } + + long cost = computeCost.apply(p); + if (cost < bestCost) { + bestCost = cost; + bestP = p; + } + } + } + + // 分段3：大p值（65+），使用关键值搜索+剪枝 + if (maxPackSize > 64) { + // 关键值：n的约数、接近n约数的值、质数、2的幂次 + Set keyValues = new HashSet<>(); + + // 添加n的约数 + for (int d = 1; d * d <= n && d <= maxPackSize; d++) { + if (n % d == 0) { + if (d > 64 && d <= maxPackSize) keyValues.add(d); + int other = n / d; + if (other > 64 && other <= maxPackSize) keyValues.add(other); + } + } + + // 添加质数（每隔几个质数） + int primeCount = 0; + for (int i = 65; i <= maxPackSize; i++) { +// if (isPrime(i)) { +// primeCount++; +// if (primeCount % 3 == 0) { // 每隔3个质数取一个 +// keyValues.add(i); +// } +// } + } + + // 添加2的幂次 + for (int pow = 7; (1 << pow) <= maxPackSize; pow++) { + int p = 1 << pow; + if (p > 64) keyValues.add(p); + } + + // 在关键值中搜索 + for (int p : keyValues) { + if (p > maxPackSize) continue; + + long lowerBound = computeLowerBound.apply(p); + if (lowerBound >= bestCost) { + continue; + } + + long cost = computeCost.apply(p); + if (cost < bestCost) { + bestCost = cost; + bestP = p; + } + } + } + + // 4. 最终验证：在找到的最优解附近进行局部搜索，确保没有错过最优解 + // 局部搜索范围：以bestP为中心，左右各搜索10个点 + int localSearchRadius = Math.min(10, maxPackSize / 2); + int localStart = Math.max(1, bestP - localSearchRadius); + int localEnd = Math.min(maxPackSize, bestP + localSearchRadius); + + // 先计算所有点的下界，筛选出可能更好的点 + List promisingPoints = new ArrayList<>(); + for (int p = localStart; p <= localEnd; p++) { + if (p == bestP) continue; // 已经计算过了 + + long lowerBound = computeLowerBound.apply(p); + if (lowerBound < bestCost) { + promisingPoints.add(p); + } + } + + // 只计算有希望的点 + for (int p : promisingPoints) { + long cost = computeCost.apply(p); + if (cost < bestCost) { + bestCost = cost; + bestP = p; + } + } + + return bestP; + } + + // 硬编码的prev数组，基于sequences中长度>=3的序列 +// private static final int[] PREV_ARRAY = new int[1025]; // 索引0不使用 + + + private static final int[] PREV_ARRAY = { + 0, 0, 1, 0, 2, 0, 3, 0, 4, 3, 5, 0, 6, 0, 7, 5, 8, 0, 9, 0, 10, 7, 11, 0, 12, 5, 13, 9, 14, 0, 15, 0, 16, 11, 17, 0, 18, 0, 19, 13, 20, 0, 21, 0, 22, 15, 23, 0, 24, 7, 25, 17, 26, 0, 27, 11, 28, 19, 29, 0, 30, 0, + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61 + 31, 21, 32, 0, 33, 0, 34, 23, 35, 0, 36, 0, 37, 0, 38, 0, 39, 0, 40, 27, 41, 0, 42, 17, 43, 29, 44, 0, 45, 0, 46, 31, 47, 19, 48, 0, 49, 33, 50, 0, 51, 0, 52, 0, 53, 0, 54, 0, 55, 37, 56, 0, 57, 23, 58, 39, 59, 0, 60, 0, 61, 41, + // 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123 + 62, 25, 63, 0, 64, 43, 65, 0, 66, 0, 67, 45, 68, 0, 69, 0, 70, 47, 71, 0, 72, 0, 73, 0, 74, 0, 75, 0, 76, 51, 77, 0, 78, 0, 79, 53, 80, 0, 81, 0, 82, 0, 83, 0, 84, 0, 85, 57, 86, 0, 87, 0, 88, 59, 89, 0, 90, 0, 91, 0, 92, 0, + // 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185 + 93, 0, 94, 63, 95, 0, 96, 0, 97, 0, 98, 0, 99, 0, 100, 0, 101, 0, 102, 0, 103, 69, 104, 0, 105, 0, 106, 0, 107, 0, 108, 0, 109, 0, 110, 0, 111, 0, 112, 0, 113, 0, 114, 0, 115, 0, 116, 0, 117, 0, 118, 0, 119, 0, 120, 0, 121, 81, 122, 0, 123, 0, + // 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247 + 124, 0, 125, 0, 126, 0, 127, 0, 128, 0, 129, 0, 130, 87, 131, 0, 132, 0, 133, 0, 134, 0, 135, 0, 136, 0, 137, 55, 138, 0, 139, 93, 140, 0, 141, 0, 142, 0, 143, 0, 144, 0, 145, 0, 146, 0, 147, 0, 148, 99, 149, 0, 150, 0, 151, 0, 152, 0, 153, 0, 154, 0, + // 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309 + 155, 0, 156, 0, 157, 0, 158, 0, 159, 0, 160, 0, 161, 0, 162, 0, 163, 0, 164, 0, 165, 0, 166, 111, 167, 0, 168, 0, 169, 49, 170, 0, 171, 0, 172, 0, 173, 0, 174, 0, 175, 117, 176, 0, 177, 0, 178, 0, 179, 0, 180, 0, 181, 0, 182, 0, 183, 0, 184, 123, 185, 0, + // 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371 + 186, 0, 187, 0, 188, 0, 189, 0, 190, 0, 191, 0, 192, 0, 193, 129, 194, 0, 195, 0, 196, 0, 197, 0, 198, 0, 199, 0, 200, 0, 201, 0, 202, 135, 203, 0, 204, 0, 205, 0, 206, 0, 207, 0, 208, 0, 209, 0, 210, 0, 211, 141, 212, 85, 213, 0, 214, 0, 215, 0, 216, 0, + // 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433 + 217, 0, 218, 0, 219, 0, 220, 0, 221, 0, 222, 0, 223, 0, 224, 0, 225, 0, 226, 0, 227, 0, 228, 0, 229, 153, 230, 0, 231, 0, 232, 0, 233, 0, 234, 0, 235, 0, 236, 0, 237, 95, 238, 159, 239, 0, 240, 0, 241, 0, 242, 0, 243, 0, 244, 0, 245, 0, 246, 0, 247, 0, + // 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495 + 248, 0, 249, 0, 250, 0, 251, 0, 252, 0, 253, 0, 254, 0, 255, 0, 256, 171, 257, 0, 258, 0, 259, 0, 260, 0, 261, 0, 262, 0, 263, 0, 264, 0, 265, 177, 266, 0, 267, 0, 268, 0, 269, 0, 270, 0, 271, 0, 272, 0, 273, 0, 274, 0, 275, 0, 276, 0, 277, 0, 278, 0, + // 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557 + 279, 0, 280, 0, 281, 0, 282, 0, 283, 189, 284, 0, 285, 0, 286, 0, 287, 115, 288, 0, 289, 0, 290, 0, 291, 0, 292, 0, 293, 0, 294, 0, 295, 0, 296, 0, 297, 0, 298, 0, 299, 0, 300, 0, 301, 0, 302, 0, 303, 0, 304, 0, 305, 0, 306, 0, 307, 0, 308, 0, 309, 0, + // 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619 + 310, 0, 311, 0, 312, 0, 313, 0, 314, 0, 315, 0, 316, 0, 317, 0, 318, 0, 319, 0, 320, 0, 321, 0, 322, 0, 323, 0, 324, 0, 325, 0, 326, 0, 327, 0, 328, 0, 329, 0, 330, 0, 331, 0, 332, 0, 333, 0, 334, 0, 335, 0, 336, 0, 337, 0, 338, 0, 339, 0, 340, 0, + // 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681 + 341, 0, 342, 0, 343, 0, 344, 0, 345, 0, 346, 0, 347, 0, 348, 0, 349, 0, 350, 0, 351, 0, 352, 0, 353, 0, 354, 0, 355, 0, 356, 0, 357, 0, 358, 0, 359, 0, 360, 0, 361, 0, 362, 0, 363, 0, 364, 243, 365, 0, 366, 0, 367, 0, 368, 0, 369, 0, 370, 0, 371, 0, + // 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743 + 372, 0, 373, 0, 374, 0, 375, 0, 376, 0, 377, 0, 378, 0, 379, 0, 380, 0, 381, 0, 382, 0, 383, 0, 384, 0, 385, 0, 386, 0, 387, 0, 388, 0, 389, 0, 390, 0, 391, 261, 392, 0, 393, 0, 394, 0, 395, 0, 396, 0, 397, 0, 398, 0, 399, 0, 400, 0, 401, 0, 402, 0, + // 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805 + 403, 0, 404, 0, 405, 0, 406, 0, 407, 0, 408, 0, 409, 0, 410, 0, 411, 0, 412, 0, 413, 0, 414, 0, 415, 0, 416, 0, 417, 0, 418, 279, 419, 0, 420, 0, 421, 0, 422, 0, 423, 0, 424, 0, 425, 0, 426, 0, 427, 0, 428, 0, 429, 0, 430, 0, 431, 0, 432, 0, 433, 0, + // 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867 + 434, 0, 435, 0, 436, 0, 437, 0, 438, 0, 439, 0, 440, 0, 441, 0, 442, 0, 443, 0, 444, 0, 445, 0, 446, 297, 447, 0, 448, 0, 449, 0, 450, 0, 451, 0, 452, 0, 453, 0, 454, 0, 455, 0, 456, 0, 457, 0, 458, 0, 459, 0, 460, 0, 461, 0, 462, 0, 463, 0, 464, 0, + // 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929 + 465, 0, 466, 0, 467, 0, 468, 0, 469, 0, 470, 0, 471, 0, 472, 0, 473, 0, 474, 0, 475, 0, 476, 0, 477, 0, 478, 0, 479, 0, 480, 0, 481, 0, 482, 0, 483, 0, 484, 0, 485, 0, 486, 0, 487, 0, 488, 0, 489, 0, 490, 0, 491, 0, 492, 0, 493, 0, 494, 0, 495, 0, + // 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991 + 496, 0, 497, 0, 498, 0, 499, 333, 500, 0, 501, 0, 502, 0, 503, 0, 504, 0, 505, 0, 506, 0, 507, 0, 508, 0, 509, 0, 510, 0, 511, 0, 512, 0, + // 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024 + 513,0,514,0,515,0,516,0,517,0,518,0,519,0,520,0,521,0,522,0,523,0,524,0,525,0,526,0,527,0,528,0,529,0,530,0,531,0,532,0,533,0,534,0,535,0,536,0,537,0,538,0,539,0,540,0,541,0,542,0,543,0,544,0,545,0,546,0,547,0,548,0,549,0,550,0,551,0,552,0,553,0,554,0,555,0,556,0,557,0,558,0,559,0,560,0,561,0,562,0,563,0,564,0,565,0,566,0,567,0,568,0,569,0,570,0,571,0,572,0,573,0,574,0,575,0,576,0,577,0,578,0,579,0,580,0,581,0,582,0,583,0,584,0,585,0,586,0,587,0,588,0,589,0,590,0,591,0,592,0,593,0,594,0,595,0,596,0,597,0,598,0,599,0,600,0,601,0,602,0,603,0,604,0,605,0,606,0,607,0,608,0,609,0,610,0,611,0,612,0,613,0,614,0,615,0,616,0,617,0,618,0,619,0,620,0,621,0,622,0,623,0,624,0,625,0,626,0,627,0,628,0,629,0,630,0,631,0,632,0,633,0,634,0,635,0,636,0,637,0,638,0,639,0,640,0,641,0,642,0,643,0,644,0,645,0,646,0,647,0,648,0,649,0,650,0,651,0,652,0,653,0,654,0,655,0,656,0,657,0,658,0,659,0,660,0,661,0,662,0,663,0,664,0,665,0,666,0,667,0,668,0,669,0,670,0,671,0,672,0,673,0,674,0,675,0,676,0,677,0,678,0,679,0,680,0,681,0,682,0,683,0,684,0,685,0,686,0,687,0,688,0,689,0,690,0,691,0,692,0,693,0,694,0,695,0,696,0,697,0,698,0,699,0,700,0,701,0,702,0,703,0,704,0,705,0,706,0,707,0,708,0,709,0,710,0,711,0,712,0,713,0,714,0,715,0,716,0,717,0,718,0,719,0,720,0,721,0,722,0,723,0,724,0,725,0,726,0,727,0,728,0,729,0,730,0,731,0,732,0,733,0,734,0,735,0,736,0,737,0,738,0,739,0,740,0,741,0,742,0,743,0,744,0,745,0,746,0,747,0,748,0,749,0,750,0,751,0,752,0,753,0,754,0,755,0,756,0,757,0,758,0,759,0,760,0,761,0,762,0,763,0,764,0,765,0,766,0,767,0,768,0,769,0,770,0,771,0,772,0,773,0,774,0,775,0,776,0,777,0,778,0,779,0,780,0,781,0,782,0,783,0,784,0,785,0,786, + 0,787,0,788,0,789,0,790,0,791,0,792,0,793,0,794,0,795,0,796,0,797,0,798,0,799,0,800,0,801,0,802,0,803,0,804,0,805,0,806,0,807,0,808,0,809,0,810,0,811,0,812,0,813,0,814,0,815,0,816,0,817,0,818,0,819,0,820,0,821,0,822,0,823,0,824,0,825,0,826,0,827,0,828,0,829,0,830,0,831,0,832,0,833,0,834,0,835,0,836,0,837,0,838,0,839,0,840,0,841,0,842,0,843,0,844,0,845,0,846,0,847,0,848,0,849,0,850,0,851,0,852,0,853,0,854,0,855,0,856,0,857,0,858,0,859,0,860,0,861,0,862,0,863,0,864,0,865,0,866,0,867,0,868,0,869,0,870,0,871,0,872,0,873,0,874,0,875,0,876,0,877,0,878,0,879,0,880,0,881,0,882,0,883,0,884,0,885,0,886,0,887,0,888,0,889,0,890,0,891,0,892,0,893,0,894,0,895,0,896,0,897,0,898,0,899,0,900,0,901,0,902,0,903,0,904,0,905,0,906,0,907,0,908,0,909,0,910,0,911,0,912,0,913,0,914,0,915,0,916,0,917,0,918,0,919,0,920,0,921,0,922,0,923,0,924,0,925,0,926,0,927,0,928,0,929,0,930,0,931,0,932,0,933,0,934,0,935,0,936,0,937,0,938,0,939,0,940,0,941,0,942,0,943,0,944,0,945,0,946,0,947,0,948,0,949,0,950,0,951,0,952,0,953,0,954,0,955,0,956,0,957,0,958,0,959,0,960,0,961,0,962,0,963,0,964,0,965,0,966,0,967,0,968,0,969,0,970,0,971,0,972,0,973,0,974,0,975,0,976,0,977,0,978,0,979,0,980,0,981,0,982,0,983,0,984,0,985,0,986,0,987,0,988,0,989,0,990,0,991,0,992,0,993,0,994,0,995,0,996,0,997,0,998,0,999,0,1000,0,1001,0,1002,0,1003,0,1004,0,1005,0,1006,0,1007,0,1008,0,1009,0,1010,0,1011,0,1012,0,1013,0,1014,0,1015,0,1016,0,1017,0,1018,0,1019,0,1020,0,1021,0,1022,0,1023,0, + 1024,0,1025,0,1026,0,1027,0,1028,0,1029,0,1030,0,1031,0,1032,0,1033,0,1034,0,1035,0,1036,0,1037,0,1038,0,1039,0,1040,0,1041,0,1042,0,1043,0,1044,0,1045,0,1046,0,1047,0,1048,0,1049,0,1050,0,1051,0,1052,0,1053,0,1054,0,1055,0,1056,0,1057,0,1058,0,1059,0,1060,0,1061,0,1062,0,1063,0,1064,0,1065,0,1066,0,1067,0,1068,0,1069,0,1070,0,1071,0,1072,0,1073,0,1074,0,1075,0,1076,0,1077,0,1078,0,1079,0,1080,0,1081,0,1082,0,1083,0,1084,0,1085,0,1086,0,1087,0,1088,0,1089,0,1090,0,1091,0,1092,0,1093,0,1094,0,1095,0,1096,0,1097,0,1098,0,1099,0,1100,0,1101,0,1102,0,1103,0,1104,0,1105,0,1106,0,1107,0,1108,0,1109,0,1110,0,1111,0,1112,0,1113,0,1114,0,1115,0,1116,0,1117,0,1118,0,1119,0,1120,0,1121,0,1122,0,1123,0,1124,0,1125,0,1126,0,1127,0,1128,0,1129,0,1130,0,1131,0,1132,0,1133,0,1134,0,1135,0,1136,0,1137,0,1138,0,1139,0,1140,0,1141,0,1142,0,1143,0,1144,0,1145,0,1146,0,1147,0,1148,0,1149,0,1150,0,1151,0,1152,0,1153,0,1154,0,1155,0,1156,0,1157,0,1158,0,1159,0,1160,0,1161,0,1162,0,1163,0,1164,0,1165,0,1166,0,1167,0,1168,0,1169,0,1170,0,1171,0,1172,0,1173,0,1174,0,1175,0,1176,0,1177,0,1178,0,1179,0,1180,0,1181,0,1182,0,1183,0,1184,0,1185,0,1186,0,1187,0,1188,0,1189,0,1190,0,1191,0,1192,0,1193,0,1194,0,1195,0,1196,0,1197,0,1198,0,1199,0,1200,0,1201,0,1202,0,1203,0,1204,0,1205,0,1206,0,1207,0,1208,0,1209,0,1210,0,1211,0,1212,0,1213,0,1214,0,1215,0,1216,0,1217,0,1218,0,1219,0,1220,0,1221,0,1222,0,1223,0,1224,0,1225,0,1226,0,1227,0,1228,0,1229,0,1230,0,1231,0,1232,0,1233,0,1234,0,1235,0,1236,0,1237,0,1238,0,1239,0,1240,0,1241,0,1242,0,1243,0,1244,0,1245,0,1246,0,1247,0,1248,0,1249,0,1250,0,1251,0,1252,0,1253,0,1254,0,1255,0,1256, + 0,1257,0,1258,0,1259,0,1260,0,1261,0,1262,0,1263,0,1264,0,1265,0,1266,0,1267,0,1268,0,1269,0,1270,0,1271,0,1272,0,1273,0,1274,0,1275,0,1276,0,1277,0,1278,0,1279,0,1280,0,1281,0,1282,0,1283,0,1284,0,1285,0,1286,0,1287,0,1288,0,1289,0,1290,0,1291,0,1292,0,1293,0,1294,0,1295,0,1296,0,1297,0,1298,0,1299,0,1300,0,1301,0,1302,0,1303,0,1304,0,1305,0,1306,0,1307,0,1308,0,1309,0,1310,0,1311,0,1312,0,1313,0,1314,0,1315,0,1316,0,1317,0,1318,0,1319,0,1320,0,1321,0,1322,0,1323,0,1324,0,1325,0,1326,0,1327,0,1328,0,1329,0,1330,0,1331,0,1332,0,1333,0,1334,0,1335,0,1336,0,1337,0,1338,0,1339,0,1340,0,1341,0,1342,0,1343,0,1344,0,1345,0,1346,0,1347,0,1348,0,1349,0,1350,0,1351,0,1352,0,1353,0,1354,0,1355,0,1356,0,1357,0,1358,0,1359,0,1360,0,1361,0,1362,0,1363,0,1364,0,1365,0,1366,0,1367,0,1368,0,1369,0,1370,0,1371,0,1372,0,1373,0,1374,0,1375,0,1376,0,1377,0,1378,0,1379,0,1380,0,1381,0,1382,0,1383,0,1384,0,1385,0,1386,0,1387,0,1388,0,1389,0,1390,0,1391,0,1392,0,1393,0,1394,0,1395,0,1396,0,1397,0,1398,0,1399,0,1400,0,1401,0,1402,0,1403,0,1404,0,1405,0,1406,0,1407,0,1408,0,1409,0,1410,0,1411,0,1412,0,1413,0,1414,0,1415,0,1416,0,1417,0,1418,0,1419,0,1420,0,1421,0,1422,0,1423,0,1424,0,1425,0,1426,0,1427,0,1428,0,1429,0,1430,0,1431,0,1432,0,1433,0,1434,0,1435,0,1436,0,1437,0,1438,0,1439,0,1440,0,1441,0,1442,0,1443,0,1444,0,1445,0,1446,0,1447,0,1448,0,1449,0,1450,0,1451,0,1452,0,1453,0,1454,0,1455,0,1456,0,1457,0,1458,0,1459,0,1460,0,1461,0,1462,0,1463,0,1464,0,1465,0,1466,0,1467,0,1468,0,1469,0,1470,0,1471,0,1472,0,1473,0,1474,0,1475,0,1476,0,1477,0,1478,0,1479,0,1480,0,1481,0,1482,0,1483,0,1484,0,1485,0,1486,0,1487,0,1488,0,1489,0,1490,0,1491,0,1492,0,1493,0,1494,0,1495,0,1496,0,1497,0,1498,0,1499,0,1500,0,1501,0,1502,0,1503,0,1504,0,1505,0,1506,0,1507,0,1508,0,1509,0,1510,0,1511,0,1512,0,1513,0,1514,0,1515,0,1516,0,1517,0,1518,0,1519,0,1520,0,1521,0,1522,0,1523,0,1524,0,1525,0,1526,0,1527,0,1528,0,1529,0,1530,0,1531,0,1532,0,1533,0,1534,0,1535,0, + 1536,0,1537,0,1538,0,1539,0,1540,0,1541,0,1542,0,1543,0,1544,0,1545,0,1546,0,1547,0,1548,0,1549,0,1550,0,1551,0,1552,0,1553,0,1554,0,1555,0,1556,0,1557,0,1558,0,1559,0,1560,0,1561,0,1562,0,1563,0,1564,0,1565,0,1566,0,1567,0,1568,0,1569,0,1570,0,1571,0,1572,0,1573,0,1574,0,1575,0,1576,0,1577,0,1578,0,1579,0,1580,0,1581,0,1582,0,1583,0,1584,0,1585,0,1586,0,1587,0,1588,0,1589,0,1590,0,1591,0,1592,0,1593,0,1594,0,1595,0,1596,0,1597,0,1598,0,1599,0,1600,0,1601,0,1602,0,1603,0,1604,0,1605,0,1606,0,1607,0,1608,0,1609,0,1610,0,1611,0,1612,0,1613,0,1614,0,1615,0,1616,0,1617,0,1618,0,1619,0,1620,0,1621,0,1622,0,1623,0,1624,0,1625,0,1626,0,1627,0,1628,0,1629,0,1630,0,1631,0,1632,0,1633,0,1634,0,1635,0,1636,0,1637,0,1638,0,1639,0,1640,0,1641,0,1642,0,1643,0,1644,0,1645,0,1646,0,1647,0,1648,0,1649,0,1650,0,1651,0,1652,0,1653,0,1654,0,1655,0,1656,0,1657,0,1658,0,1659,0,1660,0,1661,0,1662,0,1663,0,1664,0,1665,0,1666,0,1667,0,1668,0,1669,0,1670,0,1671,0,1672,0,1673,0,1674,0,1675,0,1676,0,1677,0,1678,0,1679,0,1680,0,1681,0,1682,0,1683,0,1684,0,1685,0,1686,0,1687,0,1688,0,1689,0,1690,0,1691,0,1692,0,1693,0,1694,0,1695,0,1696,0,1697,0,1698,0,1699,0,1700,0,1701,0,1702,0,1703,0,1704,0,1705,0,1706,0,1707,0,1708,0,1709,0,1710,0,1711,0,1712,0,1713,0,1714,0,1715,0,1716,0,1717,0,1718,0,1719,0,1720,0,1721,0,1722,0,1723,0,1724,0,1725,0,1726,0,1727,0,1728,0,1729,0,1730,0,1731,0,1732,0,1733,0,1734,0,1735,0,1736,0,1737,0,1738,0,1739,0,1740,0,1741,0,1742,0,1743,0,1744,0,1745,0,1746,0,1747,0,1748,0,1749,0,1750,0,1751,0,1752,0,1753,0,1754,0,1755,0,1756,0,1757,0,1758,0,1759,0,1760,0,1761,0,1762,0,1763,0,1764,0,1765,0,1766,0,1767,0,1768,0,1769,0,1770,0,1771,0,1772,0,1773,0,1774,0,1775,0,1776,0,1777,0,1778,0,1779,0,1780,0,1781,0,1782,0,1783,0,1784,0,1785,0,1786, + 0,1787,0,1788,0,1789,0,1790,0,1791,0,1792,0,1793,0,1794,0,1795,0,1796,0,1797,0,1798,0,1799,0,1800,0,1801,0,1802,0,1803,0,1804,0,1805,0,1806,0,1807,0,1808,0,1809,0,1810,0,1811,0,1812,0,1813,0,1814,0,1815,0,1816,0,1817,0,1818,0,1819,0,1820,0,1821,0,1822,0,1823,0,1824,0,1825,0,1826,0,1827,0,1828,0,1829,0,1830,0,1831,0,1832,0,1833,0,1834,0,1835,0,1836,0,1837,0,1838,0,1839,0,1840,0,1841,0,1842,0,1843,0,1844,0,1845,0,1846,0,1847,0,1848,0,1849,0,1850,0,1851,0,1852,0,1853,0,1854,0,1855,0,1856,0,1857,0,1858,0,1859,0,1860,0,1861,0,1862,0,1863,0,1864,0,1865,0,1866,0,1867,0,1868,0,1869,0,1870,0,1871,0,1872,0,1873,0,1874,0,1875,0,1876,0,1877,0,1878,0,1879,0,1880,0,1881,0,1882,0,1883,0,1884,0,1885,0,1886,0,1887,0,1888,0,1889,0,1890,0,1891,0,1892,0,1893,0,1894,0,1895,0,1896,0,1897,0,1898,0,1899,0,1900,0,1901,0,1902,0,1903,0,1904,0,1905,0,1906,0,1907,0,1908,0,1909,0,1910,0,1911,0,1912,0,1913,0,1914,0,1915,0,1916,0,1917,0,1918,0,1919,0,1920,0,1921,0,1922,0,1923,0,1924,0,1925,0,1926,0,1927,0,1928,0,1929,0,1930,0,1931,0,1932,0,1933,0,1934,0,1935,0,1936,0,1937,0,1938,0,1939,0,1940,0,1941,0,1942,0,1943,0,1944,0,1945,0,1946,0,1947,0,1948,0,1949,0,1950,0,1951,0,1952,0,1953,0,1954,0,1955,0,1956,0,1957,0,1958,0,1959,0,1960,0,1961,0,1962,0,1963,0,1964,0,1965,0,1966,0,1967,0,1968,0,1969,0,1970,0,1971,0,1972,0,1973,0,1974,0,1975,0,1976,0,1977,0,1978,0,1979,0,1980,0,1981,0,1982,0,1983,0,1984,0,1985,0,1986,0,1987,0,1988,0,1989,0,1990,0,1991,0,1992,0,1993,0,1994,0,1995,0,1996,0,1997,0,1998,0,1999,0,2000,0,2001,0,2002,0,2003,0,2004,0,2005,0,2006,0,2007,0,2008,0,2009,0,2010,0,2011,0,2012,0,2013,0,2014,0,2015,0,2016,0,2017,0,2018,0,2019,0,2020,0,2021,0,2022,0,2023,0,2024,0,2025,0,2026,0,2027,0,2028,0,2029,0,2030,0,2031,0,2032,0,2033,0,2034,0,2035,0,2036,0,2037,0,2038,0,2039,0,2040,0,2041,0,2042,0,2043,0,2044,0,2045,0,2046,0,2047,0, + 2048,0,2049,0,2050,0,2051,0,2052,0,2053,0,2054,0,2055,0,2056,0,2057,0,2058,0,2059,0,2060,0,2061,0,2062,0,2063,0,2064,0,2065,0,2066,0,2067,0,2068,0,2069,0,2070,0,2071,0,2072,0,2073,0,2074,0,2075,0,2076,0,2077,0,2078,0,2079,0,2080,0,2081,0,2082,0,2083,0,2084,0,2085,0,2086,0,2087,0,2088,0,2089,0,2090,0,2091,0,2092,0,2093,0,2094,0,2095,0,2096,0,2097,0,2098,0,2099,0,2100,0,2101,0,2102,0,2103,0,2104,0,2105,0,2106,0,2107,0,2108,0,2109,0,2110,0,2111,0,2112,0,2113,0,2114,0,2115,0,2116,0,2117,0,2118,0,2119,0,2120,0,2121,0,2122,0,2123,0,2124,0,2125,0,2126,0,2127,0,2128,0,2129,0,2130,0,2131,0,2132,0,2133,0,2134,0,2135,0,2136,0,2137,0,2138,0,2139,0,2140,0,2141,0,2142,0,2143,0,2144,0,2145,0,2146,0,2147,0,2148,0,2149,0,2150,0,2151,0,2152,0,2153,0,2154,0,2155,0,2156,0,2157,0,2158,0,2159,0,2160,0,2161,0,2162,0,2163,0,2164,0,2165,0,2166,0,2167,0,2168,0,2169,0,2170,0,2171,0,2172,0,2173,0,2174,0,2175,0,2176,0,2177,0,2178,0,2179,0,2180,0,2181,0,2182,0,2183,0,2184,0,2185,0,2186,0,2187,0,2188,0,2189,0,2190,0,2191,0,2192,0,2193,0,2194,0,2195,0,2196,0,2197,0,2198,0,2199,0,2200,0,2201,0,2202,0,2203,0,2204,0,2205,0,2206,0,2207,0,2208,0,2209,0,2210,0,2211,0,2212,0,2213,0,2214,0,2215,0,2216,0,2217,0,2218,0,2219,0,2220,0,2221,0,2222,0,2223,0,2224,0,2225,0,2226,0,2227,0,2228,0,2229,0,2230,0,2231,0,2232,0,2233,0,2234,0,2235,0,2236,0,2237,0,2238,0,2239,0,2240,0,2241,0,2242,0,2243,0,2244,0,2245,0,2246,0,2247,0,2248,0,2249,0,2250,0,2251,0,2252,0,2253,0,2254,0,2255,0,2256, + 0,2257,0,2258,0,2259,0,2260,0,2261,0,2262,0,2263,0,2264,0,2265,0,2266,0,2267,0,2268,0,2269,0,2270,0,2271,0,2272,0,2273,0,2274,0,2275,0,2276,0,2277,0,2278,0,2279,0,2280,0,2281,0,2282,0,2283,0,2284,0,2285,0,2286,0,2287,0,2288,0,2289,0,2290,0,2291,0,2292,0,2293,0,2294,0,2295,0,2296,0,2297,0,2298,0,2299,0,2300,0,2301,0,2302,0,2303,0,2304,0,2305,0,2306,0,2307,0,2308,0,2309,0,2310,0,2311,0,2312,0,2313,0,2314,0,2315,0,2316,0,2317,0,2318,0,2319,0,2320,0,2321,0,2322,0,2323,0,2324,0,2325,0,2326,0,2327,0,2328,0,2329,0,2330,0,2331,0,2332,0,2333,0,2334,0,2335,0,2336,0,2337,0,2338,0,2339,0,2340,0,2341,0,2342,0,2343,0,2344,0,2345,0,2346,0,2347,0,2348,0,2349,0,2350,0,2351,0,2352,0,2353,0,2354,0,2355,0,2356,0,2357,0,2358,0,2359,0,2360,0,2361,0,2362,0,2363,0,2364,0,2365,0,2366,0,2367,0,2368,0,2369,0,2370,0,2371,0,2372,0,2373,0,2374,0,2375,0,2376,0,2377,0,2378,0,2379,0,2380,0,2381,0,2382,0,2383,0,2384,0,2385,0,2386,0,2387,0,2388,0,2389,0,2390,0,2391,0,2392,0,2393,0,2394,0,2395,0,2396,0,2397,0,2398,0,2399,0,2400,0,2401,0,2402,0,2403,0,2404,0,2405,0,2406,0,2407,0,2408,0,2409,0,2410,0,2411,0,2412,0,2413,0,2414,0,2415,0,2416,0,2417,0,2418,0,2419,0,2420,0,2421,0,2422,0,2423,0,2424,0,2425,0,2426,0,2427,0,2428,0,2429,0,2430,0,2431,0,2432,0,2433,0,2434,0,2435,0,2436,0,2437,0,2438,0,2439,0,2440,0,2441,0,2442,0,2443,0,2444,0,2445,0,2446,0,2447,0,2448,0,2449,0,2450,0,2451,0,2452,0,2453,0,2454,0,2455,0,2456,0,2457,0,2458,0,2459,0,2460,0,2461,0,2462,0,2463,0,2464,0,2465,0,2466,0,2467,0,2468,0,2469,0,2470,0,2471,0,2472,0,2473,0,2474,0,2475,0,2476,0,2477,0,2478,0,2479,0,2480,0,2481,0,2482,0,2483,0,2484,0,2485,0,2486,0,2487,0,2488,0,2489,0,2490,0,2491,0,2492,0,2493,0,2494,0,2495,0,2496,0,2497,0,2498,0,2499,0,2500,0,2501,0,2502,0,2503,0,2504,0,2505,0,2506,0,2507,0,2508,0,2509,0,2510,0,2511,0,2512,0,2513,0,2514,0,2515,0,2516,0,2517,0,2518,0,2519,0,2520,0,2521,0,2522,0,2523,0,2524,0,2525,0,2526,0,2527,0,2528,0,2529,0,2530,0,2531,0,2532,0,2533,0,2534,0,2535,0,2536,0,2537,0,2538,0,2539,0,2540,0,2541,0,2542,0,2543,0,2544,0,2545,0,2546,0,2547,0,2548,0,2549,0,2550,0,2551,0,2552,0,2553,0,2554,0,2555,0,2556,0,2557,0,2558,0,2559,0, + 2560,0,2561,0,2562,0,2563,0,2564,0,2565,0,2566,0,2567,0,2568,0,2569,0,2570,0,2571,0,2572,0,2573,0,2574,0,2575,0,2576,0,2577,0,2578,0,2579,0,2580,0,2581,0,2582,0,2583,0,2584,0,2585,0,2586,0,2587,0,2588,0,2589,0,2590,0,2591,0,2592,0,2593,0,2594,0,2595,0,2596,0,2597,0,2598,0,2599,0,2600,0,2601,0,2602,0,2603,0,2604,0,2605,0,2606,0,2607,0,2608,0,2609,0,2610,0,2611,0,2612,0,2613,0,2614,0,2615,0,2616,0,2617,0,2618,0,2619,0,2620,0,2621,0,2622,0,2623,0,2624,0,2625,0,2626,0,2627,0,2628,0,2629,0,2630,0,2631,0,2632,0,2633,0,2634,0,2635,0,2636,0,2637,0,2638,0,2639,0,2640,0,2641,0,2642,0,2643,0,2644,0,2645,0,2646,0,2647,0,2648,0,2649,0,2650,0,2651,0,2652,0,2653,0,2654,0,2655,0,2656,0,2657,0,2658,0,2659,0,2660,0,2661,0,2662,0,2663,0,2664,0,2665,0,2666,0,2667,0,2668,0,2669,0,2670,0,2671,0,2672,0,2673,0,2674,0,2675,0,2676,0,2677,0,2678,0,2679,0,2680,0,2681,0,2682,0,2683,0,2684,0,2685,0,2686,0,2687,0,2688,0,2689,0,2690,0,2691,0,2692,0,2693,0,2694,0,2695,0,2696,0,2697,0,2698,0,2699,0,2700,0,2701,0,2702,0,2703,0,2704,0,2705,0,2706,0,2707,0,2708,0,2709,0,2710,0,2711,0,2712,0,2713,0,2714,0,2715,0,2716,0,2717,0,2718,0,2719,0,2720,0,2721,0,2722,0,2723,0,2724,0,2725,0,2726,0,2727,0,2728,0,2729,0,2730,0,2731,0,2732,0,2733,0,2734,0,2735,0,2736,0,2737,0,2738,0,2739,0,2740,0,2741,0,2742,0,2743,0,2744,0,2745,0,2746,0,2747,0,2748,0,2749,0,2750,0,2751,0,2752,0,2753,0,2754,0,2755,0,2756,0,2757,0,2758,0,2759,0,2760,0,2761,0,2762,0,2763,0,2764,0,2765,0,2766,0,2767,0,2768,0,2769,0,2770,0,2771,0,2772,0,2773,0,2774,0,2775,0,2776,0,2777,0,2778,0,2779,0,2780,0,2781,0,2782,0,2783,0,2784,0,2785,0,2786, + 0,2787,0,2788,0,2789,0,2790,0,2791,0,2792,0,2793,0,2794,0,2795,0,2796,0,2797,0,2798,0,2799,0,2800,0,2801,0,2802,0,2803,0,2804,0,2805,0,2806,0,2807,0,2808,0,2809,0,2810,0,2811,0,2812,0,2813,0,2814,0,2815,0,2816,0,2817,0,2818,0,2819,0,2820,0,2821,0,2822,0,2823,0,2824,0,2825,0,2826,0,2827,0,2828,0,2829,0,2830,0,2831,0,2832,0,2833,0,2834,0,2835,0,2836,0,2837,0,2838,0,2839,0,2840,0,2841,0,2842,0,2843,0,2844,0,2845,0,2846,0,2847,0,2848,0,2849,0,2850,0,2851,0,2852,0,2853,0,2854,0,2855,0,2856,0,2857,0,2858,0,2859,0,2860,0,2861,0,2862,0,2863,0,2864,0,2865,0,2866,0,2867,0,2868,0,2869,0,2870,0,2871,0,2872,0,2873,0,2874,0,2875,0,2876,0,2877,0,2878,0,2879,0,2880,0,2881,0,2882,0,2883,0,2884,0,2885,0,2886,0,2887,0,2888,0,2889,0,2890,0,2891,0,2892,0,2893,0,2894,0,2895,0,2896,0,2897,0,2898,0,2899,0,2900,0,2901,0,2902,0,2903,0,2904,0,2905,0,2906,0,2907,0,2908,0,2909,0,2910,0,2911,0,2912,0,2913,0,2914,0,2915,0,2916,0,2917,0,2918,0,2919,0,2920,0,2921,0,2922,0,2923,0,2924,0,2925,0,2926,0,2927,0,2928,0,2929,0,2930,0,2931,0,2932,0,2933,0,2934,0,2935,0,2936,0,2937,0,2938,0,2939,0,2940,0,2941,0,2942,0,2943,0,2944,0,2945,0,2946,0,2947,0,2948,0,2949,0,2950,0,2951,0,2952,0,2953,0,2954,0,2955,0,2956,0,2957,0,2958,0,2959,0,2960,0,2961,0,2962,0,2963,0,2964,0,2965,0,2966,0,2967,0,2968,0,2969,0,2970,0,2971,0,2972,0,2973,0,2974,0,2975,0,2976,0,2977,0,2978,0,2979,0,2980,0,2981,0,2982,0,2983,0,2984,0,2985,0,2986,0,2987,0,2988,0,2989,0,2990,0,2991,0,2992,0,2993,0,2994,0,2995,0,2996,0,2997,0,2998,0,2999,0,3000,0,3001,0,3002,0,3003,0,3004,0,3005,0,3006,0,3007,0,3008,0,3009,0,3010,0,3011,0,3012,0,3013,0,3014,0,3015,0,3016,0,3017,0,3018,0,3019,0,3020,0,3021,0,3022,0,3023,0,3024,0,3025,0,3026,0,3027,0,3028,0,3029,0,3030,0,3031,0,3032,0,3033,0,3034,0,3035,0,3036,0,3037,0,3038,0,3039,0,3040,0,3041,0,3042,0,3043,0,3044,0,3045,0,3046,0,3047,0,3048,0,3049,0,3050,0,3051,0,3052,0,3053,0,3054,0,3055,0,3056,0,3057,0,3058,0,3059,0,3060,0,3061,0,3062,0,3063,0,3064,0,3065,0,3066,0,3067,0,3068,0,3069,0,3070,0,3071,0, + 3072,0,3073,0,3074,0,3075,0,3076,0,3077,0,3078,0,3079,0,3080,0,3081,0,3082,0,3083,0,3084,0,3085,0,3086,0,3087,0,3088,0,3089,0,3090,0,3091,0,3092,0,3093,0,3094,0,3095,0,3096,0,3097,0,3098,0,3099,0,3100,0,3101,0,3102,0,3103,0,3104,0,3105,0,3106,0,3107,0,3108,0,3109,0,3110,0,3111,0,3112,0,3113,0,3114,0,3115,0,3116,0,3117,0,3118,0,3119,0,3120,0,3121,0,3122,0,3123,0,3124,0,3125,0,3126,0,3127,0,3128,0,3129,0,3130,0,3131,0,3132,0,3133,0,3134,0,3135,0,3136,0,3137,0,3138,0,3139,0,3140,0,3141,0,3142,0,3143,0,3144,0,3145,0,3146,0,3147,0,3148,0,3149,0,3150,0,3151,0,3152,0,3153,0,3154,0,3155,0,3156,0,3157,0,3158,0,3159,0,3160,0,3161,0,3162,0,3163,0,3164,0,3165,0,3166,0,3167,0,3168,0,3169,0,3170,0,3171,0,3172,0,3173,0,3174,0,3175,0,3176,0,3177,0,3178,0,3179,0,3180,0,3181,0,3182,0,3183,0,3184,0,3185,0,3186,0,3187,0,3188,0,3189,0,3190,0,3191,0,3192,0,3193,0,3194,0,3195,0,3196,0,3197,0,3198,0,3199,0,3200,0,3201,0,3202,0,3203,0,3204,0,3205,0,3206,0,3207,0,3208,0,3209,0,3210,0,3211,0,3212,0,3213,0,3214,0,3215,0,3216,0,3217,0,3218,0,3219,0,3220,0,3221,0,3222,0,3223,0,3224,0,3225,0,3226,0,3227,0,3228,0,3229,0,3230,0,3231,0,3232,0,3233,0,3234,0,3235,0,3236,0,3237,0,3238,0,3239,0,3240,0,3241,0,3242,0,3243,0,3244,0,3245,0,3246,0,3247,0,3248,0,3249,0,3250,0,3251,0,3252,0,3253,0,3254,0,3255,0,3256, + 0,3257,0,3258,0,3259,0,3260,0,3261,0,3262,0,3263,0,3264,0,3265,0,3266,0,3267,0,3268,0,3269,0,3270,0,3271,0,3272,0,3273,0,3274,0,3275,0,3276,0,3277,0,3278,0,3279,0,3280,0,3281,0,3282,0,3283,0,3284,0,3285,0,3286,0,3287,0,3288,0,3289,0,3290,0,3291,0,3292,0,3293,0,3294,0,3295,0,3296,0,3297,0,3298,0,3299,0,3300,0,3301,0,3302,0,3303,0,3304,0,3305,0,3306,0,3307,0,3308,0,3309,0,3310,0,3311,0,3312,0,3313,0,3314,0,3315,0,3316,0,3317,0,3318,0,3319,0,3320,0,3321,0,3322,0,3323,0,3324,0,3325,0,3326,0,3327,0,3328,0,3329,0,3330,0,3331,0,3332,0,3333,0,3334,0,3335,0,3336,0,3337,0,3338,0,3339,0,3340,0,3341,0,3342,0,3343,0,3344,0,3345,0,3346,0,3347,0,3348,0,3349,0,3350,0,3351,0,3352,0,3353,0,3354,0,3355,0,3356,0,3357,0,3358,0,3359,0,3360,0,3361,0,3362,0,3363,0,3364,0,3365,0,3366,0,3367,0,3368,0,3369,0,3370,0,3371,0,3372,0,3373,0,3374,0,3375,0,3376,0,3377,0,3378,0,3379,0,3380,0,3381,0,3382,0,3383,0,3384,0,3385,0,3386,0,3387,0,3388,0,3389,0,3390,0,3391,0,3392,0,3393,0,3394,0,3395,0,3396,0,3397,0,3398,0,3399,0,3400,0,3401,0,3402,0,3403,0,3404,0,3405,0,3406,0,3407,0,3408,0,3409,0,3410,0,3411,0,3412,0,3413,0,3414,0,3415,0,3416,0,3417,0,3418,0,3419,0,3420,0,3421,0,3422,0,3423,0,3424,0,3425,0,3426,0,3427,0,3428,0,3429,0,3430,0,3431,0,3432,0,3433,0,3434,0,3435,0,3436,0,3437,0,3438,0,3439,0,3440,0,3441,0,3442,0,3443,0,3444,0,3445,0,3446,0,3447,0,3448,0,3449,0,3450,0,3451,0,3452,0,3453,0,3454,0,3455,0,3456,0,3457,0,3458,0,3459,0,3460,0,3461,0,3462,0,3463,0,3464,0,3465,0,3466,0,3467,0,3468,0,3469,0,3470,0,3471,0,3472,0,3473,0,3474,0,3475,0,3476,0,3477,0,3478,0,3479,0,3480,0,3481,0,3482,0,3483,0,3484,0,3485,0,3486,0,3487,0,3488,0,3489,0,3490,0,3491,0,3492,0,3493,0,3494,0,3495,0,3496,0,3497,0,3498,0,3499,0,3500,0,3501,0,3502,0,3503,0,3504,0,3505,0,3506,0,3507,0,3508,0,3509,0,3510,0,3511,0,3512,0,3513,0,3514,0,3515,0,3516,0,3517,0,3518,0,3519,0,3520,0,3521,0,3522,0,3523,0,3524,0,3525,0,3526,0,3527,0,3528,0,3529,0,3530,0,3531,0,3532,0,3533,0,3534,0,3535,0,3536,0,3537,0,3538,0,3539,0,3540,0,3541,0,3542,0,3543,0,3544,0,3545,0,3546,0,3547,0,3548,0,3549,0,3550,0,3551,0,3552,0,3553,0,3554,0,3555,0,3556,0,3557,0,3558,0,3559,0,3560,0,3561,0,3562,0,3563,0,3564,0,3565,0,3566,0,3567,0,3568,0,3569,0,3570,0,3571,0,3572,0,3573,0,3574,0,3575,0,3576,0,3577,0,3578,0,3579,0,3580,0,3581,0,3582,0,3583,0, + 3584,0,3585,0,3586,0,3587,0,3588,0,3589,0,3590,0,3591,0,3592,0,3593,0,3594,0,3595,0,3596,0,3597,0,3598,0,3599,0,3600,0,3601,0,3602,0,3603,0,3604,0,3605,0,3606,0,3607,0,3608,0,3609,0,3610,0,3611,0,3612,0,3613,0,3614,0,3615,0,3616,0,3617,0,3618,0,3619,0,3620,0,3621,0,3622,0,3623,0,3624,0,3625,0,3626,0,3627,0,3628,0,3629,0,3630,0,3631,0,3632,0,3633,0,3634,0,3635,0,3636,0,3637,0,3638,0,3639,0,3640,0,3641,0,3642,0,3643,0,3644,0,3645,0,3646,0,3647,0,3648,0,3649,0,3650,0,3651,0,3652,0,3653,0,3654,0,3655,0,3656,0,3657,0,3658,0,3659,0,3660,0,3661,0,3662,0,3663,0,3664,0,3665,0,3666,0,3667,0,3668,0,3669,0,3670,0,3671,0,3672,0,3673,0,3674,0,3675,0,3676,0,3677,0,3678,0,3679,0,3680,0,3681,0,3682,0,3683,0,3684,0,3685,0,3686,0,3687,0,3688,0,3689,0,3690,0,3691,0,3692,0,3693,0,3694,0,3695,0,3696,0,3697,0,3698,0,3699,0,3700,0,3701,0,3702,0,3703,0,3704,0,3705,0,3706,0,3707,0,3708,0,3709,0,3710,0,3711,0,3712,0,3713,0,3714,0,3715,0,3716,0,3717,0,3718,0,3719,0,3720,0,3721,0,3722,0,3723,0,3724,0,3725,0,3726,0,3727,0,3728,0,3729,0,3730,0,3731,0,3732,0,3733,0,3734,0,3735,0,3736,0,3737,0,3738,0,3739,0,3740,0,3741,0,3742,0,3743,0,3744,0,3745,0,3746,0,3747,0,3748,0,3749,0,3750,0,3751,0,3752,0,3753,0,3754,0,3755,0,3756,0,3757,0,3758,0,3759,0,3760,0,3761,0,3762,0,3763,0,3764,0,3765,0,3766,0,3767,0,3768,0,3769,0,3770,0,3771,0,3772,0,3773,0,3774,0,3775,0,3776,0,3777,0,3778,0,3779,0,3780,0,3781,0,3782,0,3783,0,3784,0,3785,0,3786, + 0,3787,0,3788,0,3789,0,3790,0,3791,0,3792,0,3793,0,3794,0,3795,0,3796,0,3797,0,3798,0,3799,0,3800,0,3801,0,3802,0,3803,0,3804,0,3805,0,3806,0,3807,0,3808,0,3809,0,3810,0,3811,0,3812,0,3813,0,3814,0,3815,0,3816,0,3817,0,3818,0,3819,0,3820,0,3821,0,3822,0,3823,0,3824,0,3825,0,3826,0,3827,0,3828,0,3829,0,3830,0,3831,0,3832,0,3833,0,3834,0,3835,0,3836,0,3837,0,3838,0,3839,0,3840,0,3841,0,3842,0,3843,0,3844,0,3845,0,3846,0,3847,0,3848,0,3849,0,3850,0,3851,0,3852,0,3853,0,3854,0,3855,0,3856,0,3857,0,3858,0,3859,0,3860,0,3861,0,3862,0,3863,0,3864,0,3865,0,3866,0,3867,0,3868,0,3869,0,3870,0,3871,0,3872,0,3873,0,3874,0,3875,0,3876,0,3877,0,3878,0,3879,0,3880,0,3881,0,3882,0,3883,0,3884,0,3885,0,3886,0,3887,0,3888,0,3889,0,3890,0,3891,0,3892,0,3893,0,3894,0,3895,0,3896,0,3897,0,3898,0,3899,0,3900,0,3901,0,3902,0,3903,0,3904,0,3905,0,3906,0,3907,0,3908,0,3909,0,3910,0,3911,0,3912,0,3913,0,3914,0,3915,0,3916,0,3917,0,3918,0,3919,0,3920,0,3921,0,3922,0,3923,0,3924,0,3925,0,3926,0,3927,0,3928,0,3929,0,3930,0,3931,0,3932,0,3933,0,3934,0,3935,0,3936,0,3937,0,3938,0,3939,0,3940,0,3941,0,3942,0,3943,0,3944,0,3945,0,3946,0,3947,0,3948,0,3949,0,3950,0,3951,0,3952,0,3953,0,3954,0,3955,0,3956,0,3957,0,3958,0,3959,0,3960,0,3961,0,3962,0,3963,0,3964,0,3965,0,3966,0,3967,0,3968,0,3969,0,3970,0,3971,0,3972,0,3973,0,3974,0,3975,0,3976,0,3977,0,3978,0,3979,0,3980,0,3981,0,3982,0,3983,0,3984,0,3985,0,3986,0,3987,0,3988,0,3989,0,3990,0,3991,0,3992,0,3993,0,3994,0,3995,0,3996,0,3997,0,3998,0,3999,0,4000,0,4001,0,4002,0,4003,0,4004,0,4005,0,4006,0,4007,0,4008,0,4009,0,4010,0,4011,0,4012,0,4013,0,4014,0,4015,0,4016,0,4017,0,4018,0,4019,0,4020,0,4021,0,4022,0,4023,0,4024,0,4025,0,4026,0,4027,0,4028,0,4029,0,4030,0,4031,0,4032,0,4033,0,4034,0,4035,0,4036,0,4037,0,4038,0,4039,0,4040,0,4041,0,4042,0,4043,0,4044,0,4045,0,4046,0,4047,0,4048,0,4049,0,4050,0,4051,0,4052,0,4053,0,4054,0,4055,0,4056,0,4057,0,4058,0,4059,0,4060,0,4061,0,4062,0,4063,0,4064,0,4065,0,4066,0,4067,0,4068,0,4069,0,4070,0,4071,0,4072,0,4073,0,4074,0,4075,0,4076,0,4077,0,4078,0,4079,0,4080,0,4081,0,4082,0,4083,0,4084,0,4085,0,4086,0,4087,0,4088,0,4089,0,4090,0,4091,0,4092,0,4093,0,4094,0,4095,0, + 4096 + }; + + // 剪枝+RMQ + public static int findOptimalPackSizeallV5(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + + // 初始化第一层 + System.arraycopy(bitWidths, 0, st[0], 0, n); + + // 构建稀疏表 + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = max_value_long; +// int count_prune = 0; + + for (int p = 1; p <= n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + long currentCost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 使用RMQ查询区间最大值 + int k = log2[p]; + int maxBitWidth = Math.max(st[k][start], st[k][end - (1 << k) + 1]); + + currentCost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = Math.max(st[k][lastStart], + st[k][lastEnd - (1 << k) + 1]); + currentCost += (long) r * lastMaxBitWidth; + } + } + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } +// filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + public static int testFindOptimalPackSizeallV5(int[] values, int[] filters_count) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + + // 初始化第一层 + System.arraycopy(bitWidths, 0, st[0], 0, n); + + // 构建稀疏表 + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = max_value_long; + int count_prune = 0; + + for (int p = 1; p <= n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; + count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + long currentCost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 使用RMQ查询区间最大值 + int k = log2[p]; + int maxBitWidth = Math.max(st[k][start], st[k][end - (1 << k) + 1]); + + currentCost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = Math.max(st[k][lastStart], + st[k][lastEnd - (1 << k) + 1]); + currentCost += (long) r * lastMaxBitWidth; + } + } + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + filters_count[0] = count_prune; + System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + // 只有剪枝 没有RMQ + public static int findOptimalPackSizeallV6(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = max_value_long; +// int count_prune = 0; + + for (int p = 1; p <= n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } +// filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + // 只有剪枝 没有RMQ + 大于n/2的剪枝 + public static int findOptimalPackSizeallV6Plus(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + boolean is_b_n_b_max = false; + boolean has_larger_b_n = false; + int bound_index = n-1; + int half_n = n / 2; + int LastValue = values[n-1]; + + // 判断开始小于value[n-1]的边界 和 value[n-1]是否是最大值 + for (int i = n-1;i>=half_n;i--) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + if(!has_larger_b_n){ + if(value>LastValue){ + has_larger_b_n = true; + bound_index = i; + } + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + if(globalMax == values[n-1]){ + is_b_n_b_max = true; + } + + for (int i = 0; i < half_n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = (long) n *bitWidthGlobal+z; +// int count_prune = 0; + + for (int p = 1; p <= half_n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = bitWidths[j]; + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + + if(!is_b_n_b_max){ +// int p = n-1; + long currentCost = 0; + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = 0; j < n-1; j++) { + int bitWidth = bitWidths[j]; + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) (n-1) * maxBitWidth; + currentCost += bitsNeeded; + currentCost += bitWidths[n-1]; + currentCost += 2L *z; + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = (n-1); + } + + }{ + for (int p = half_n+1; p <= bound_index; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + } +// filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + // 只有剪枝 没有RMQ + 大于n/2的剪枝 + public static int testFindOptimalPackSizeallV6Plus(int[] values, int[] filters_count) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + boolean is_b_n_b_max = false; + boolean has_larger_b_n = false; + int bound_index = n-1; + int half_n = n / 2; + int LastValue = values[n-1]; + + // 判断开始小于value[n-1]的边界 和 value[n-1]是否是最大值 + for (int i = n-1;i>=half_n;i--) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + if(!has_larger_b_n){ + if(value>LastValue){ + has_larger_b_n = true; + bound_index = i; + } + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + if(globalMax == values[n-1]){ + is_b_n_b_max = true; + } + + for (int i = 0; i < half_n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = (long) n *bitWidthGlobal+z; + int count_prune = 0; + + for (int p = 1; p <= half_n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; + count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = bitWidths[j]; + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + + if(!is_b_n_b_max){ +// int p = n-1; + long currentCost = 0; + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = 0; j < n-1; j++) { + int bitWidth = bitWidths[j]; + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) (n-1) * maxBitWidth; + currentCost += bitsNeeded; + currentCost += bitWidths[n-1]; + currentCost += 2L *z; + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = (n-1); + } + count_prune+=half_n; + }{ + for (int p = half_n+1; p <= bound_index; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = PREV_ARRAY[p]; + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + count_prune+= (n-bound_index); + } + filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + + + // 只有剪枝 没有RMQ for larger page size + public static int findOptimalPackSizeV7(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = max_value_long; +// int count_prune = 0; + + for (int p = 1; p <= n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = 0; + if(p > 1024) { + if(p % 2==0){ + prev = p / 2; + } + } else { + prev = PREV_ARRAY[p] ; + } + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + long currentCost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 0; + for (int j = start; j <= end; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + currentCost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + + int lastMaxBitWidth = 0; + for (int j = lastStart; j <= lastEnd; j++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + if (bitWidth > lastMaxBitWidth) { + lastMaxBitWidth = bitWidth; + } + } + + long bitsNeeded = (long) r * lastMaxBitWidth; + currentCost += bitsNeeded; + } + + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } +// filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + + // 剪枝+RMQ for larger page size + public static int findOptimalPackSizeallV8(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + + // 初始化第一层 + System.arraycopy(bitWidths, 0, st[0], 0, n); + + // 构建稀疏表 + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 初始化cost数组和is_increased数组 + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + + long max_value_long = Long.MAX_VALUE; + int bestPackSize = n; + long bestCost = max_value_long; +// int count_prune = 0; + + for (int p = 1; p <= n; p++) { + // 获取prev值（如果p>1024，使用计算逻辑） + int prev = 0; + if(p > 1024) { + if(p % 2==0){ + prev = p / 2; + } + } else { + prev = PREV_ARRAY[p] ; + } + + // 剪枝逻辑 + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; +// cost[p] = max_value_long; +// count_prune+=1; + continue; + } + + int m = (n + p - 1) / p; // ceil(n/p) + long currentCost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + + // 使用RMQ查询区间最大值 + int k = log2[p]; + int maxBitWidth = Math.max(st[k][start], st[k][end - (1 << k) + 1]); + + currentCost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = Math.max(st[k][lastStart], + st[k][lastEnd - (1 << k) + 1]); + currentCost += (long) r * lastMaxBitWidth; + } + } + + currentCost += (long) m * z; + cost[p] = currentCost; + + // 更新is_increased数组 + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + // 更新最优解 + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } +// filters_count[0] = count_prune; +// System.out.println("count_prune（/1024）:" + count_prune); + + return bestPackSize; + } + + + private static int[] scaleNumbers(List numbers, int decimalMax) { + BigDecimal scale = BigDecimal.TEN.pow(decimalMax); + int size = numbers.size(); + int[] result = new int[size]; + + if (size == 0) { + return result; + } + + BigDecimal min = null; + BigDecimal[] scaledValues = new BigDecimal[size]; + + for (int i = 0; i < size; i++) { + BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); + scaledValues[i] = val; + if (min == null || val.compareTo(min) < 0) { + min = val; + } + } + + BigDecimal first = scaledValues[0].subtract(min); + result[0] = first.toBigInteger().intValue(); + + for (int i = 1; i < size; i++) { + BigDecimal current = scaledValues[i].subtract(min); + result[i] = current.toBigInteger().intValue(); + } + + return result; + } + + + public static byte[] encodeBitPacking(int[] originalArray, int[] bitWidths, int pack_size) { + int totalGroups = bitWidths.length; + int n = originalArray.length; + + // 1. 计算存储bitWidths所需的位数（根据最大值动态确定） + int maxBitWidth = 0; + for (int bitWidth : bitWidths) { + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + + // 计算存储每个bitWidth需要的位数（ceil(log2(maxBitWidth+1))) + int bitsForBitWidth; + if (maxBitWidth <= 1) { + bitsForBitWidth = 1; + } else { + bitsForBitWidth = 32 - Integer.numberOfLeadingZeros(maxBitWidth); + } + + int totalBitsForBitWidths = totalGroups * bitsForBitWidth; + + // 2. 计算数据部分所需的总位数 + long totalDataBits = 0; + for (int group = 0; group < totalGroups; group++) { + int bitWidth = bitWidths[group]; + int valuesInGroup = Math.min(pack_size, n - group * pack_size); + if (valuesInGroup <= 0) continue; + + totalDataBits += (long) valuesInGroup * bitWidth; + } + + // 3. 计算总位数并分配字节数组 + long totalBits = totalBitsForBitWidths + totalDataBits; + int totalBytes = (int) ((totalBits + 7) / 8); // 向上取整到字节 + byte[] encodedResult = new byte[totalBytes]; + + // 4. 使用BitWriter来按bit写入 + BitWriter bitWriter = new BitWriter(encodedResult); + + // 5. 编码bitWidths信息（用动态位数存储） + for (int group = 0; group < totalGroups; group++) { + int bitWidth = bitWidths[group]; + bitWriter.writeBits(bitWidth, bitsForBitWidth); + } + + // 6. 编码数据部分（按bit连续存储） + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + int bitWidth = bitWidths[group]; + + int valuesInGroup = Math.min(pack_size, n - startIndex); + if (valuesInGroup <= 0) break; + + // 编码这个pack的所有值 + for (int i = 0; i < valuesInGroup; i++) { + int idx = startIndex + i; + if (idx < n) { + int value = originalArray[idx]; + // 确保value在bitWidth范围内（安全处理） + int maskedValue = value & ((1 << bitWidth) - 1); + bitWriter.writeBits(maskedValue, bitWidth); + } else { + // 填充0 + bitWriter.writeBits(0, bitWidth); + } + } + } + + // 7. 刷新剩余的bit（填充0直到字节边界） + bitWriter.flush(); + + return encodedResult; + } + + // BitWriter辅助类，用于按bit写入 + static class BitWriter { + private byte[] buffer; + private int currentByte; + private int bitPosition; // 0-7，当前字节中的bit位置 + + public BitWriter(byte[] buffer) { + this.buffer = buffer; + this.currentByte = 0; + this.bitPosition = 7; // 从最高位开始 + } + + // 写入指定数量的bits（从value的低位开始） + public void writeBits(int value, int numBits) { + if (numBits <= 0) return; + + for (int i = numBits - 1; i >= 0; i--) { + int bit = (value >> i) & 1; + + // 设置当前bit + if (bit == 1) { + buffer[currentByte] |= (1 << bitPosition); + } + + // 移动到下一个bit位置 + bitPosition--; + if (bitPosition < 0) { + // 移动到下一个字节 + currentByte++; + bitPosition = 7; + } + } + } + + // 刷新剩余的bit，用0填充直到字节边界 + public void flush() { + // 如果当前位置不是字节边界，移动到下一个字节 + if (bitPosition != 7) { + currentByte++; + bitPosition = 7; + } + } + + // 获取实际写入的字节数 + public int getBytesWritten() { + if (bitPosition == 7) { + return currentByte; + } else { + return currentByte + 1; + } + } + } + public static int[] sprintz(int[] numbers) { + int size = numbers.length; + int[] result = new int[size]; + + int first = numbers[0]; + result[0] = first; + + // 3. Process subsequent elements with delta + ZigZag encoding + int prev = first; + for (int i = 1; i < size; i++) { + int current = numbers[i]; + int diff = current - prev; + result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding + prev = current; + } + + return result; + } + + public static int[] sprintzDecode(int[] encodedData) { + int size = encodedData.length; + int[] result = new int[size]; + + if (size == 0) return result; + + // 第一个元素是原始值 + result[0] = encodedData[0]; + + // 后续元素需要ZigZag解码和累加 + int prev = result[0]; + for (int i = 1; i < size; i++) { + int zigzagEncoded = encodedData[i]; + int diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 + result[i] = prev + diff; + prev = result[i]; + } + + return result; + } + + + + public static void main(String[] args) throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV2(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + // Features + + public static double[] computeBitWidthFeatures(int[] bitWidths) { + int n = bitWidths.length; + if (n == 0) { + return new double[14]; + } + + double[] features = new double[10]; + + // 1. 位宽的平均值/64 + double sum = 0; + for (int bw : bitWidths) { + sum += bw; + } + features[0] = sum / (n * 64.0); + + // 2. 位宽的中位数/64 + int[] sortedBitWidths = bitWidths.clone(); + Arrays.sort(sortedBitWidths); + double median; + if (n % 2 == 0) { + median = (sortedBitWidths[n/2 - 1] + sortedBitWidths[n/2]) / 2.0; + } else { + median = sortedBitWidths[n/2]; + } + features[1] = median / 64.0; + +// // 3. 位宽的极差/64 +// int minBw = sortedBitWidths[0]; +// int maxBw = sortedBitWidths[n-1]; +// features[2] = (maxBw - minBw) / 64.0; + + // 4. 位宽的标准差/64 + double mean = sum / n; + double variance = 0; + for (int bw : bitWidths) { + variance += Math.pow(bw - mean, 2); + } + variance /= n; + double stdDev = Math.sqrt(variance); + features[2] = stdDev / 64.0; + + // 5-8. 位宽差分特征 + if (n > 1) { + int[] diffs = new int[n-1]; + for (int i = 0; i < n-1; i++) { + diffs[i] = Math.abs(bitWidths[i+1] - bitWidths[i]); + } + + // 5. 位宽差分绝对均值/64 + double diffSum = 0; + for (int diff : diffs) { + diffSum += diff; + } + features[3] = diffSum / ((n-1) * 64.0); + + // 6. 位宽差分中位数/64 + int[] sortedDiffs = diffs.clone(); + Arrays.sort(sortedDiffs); +// double diffMedian; +// if ((n-1) % 2 == 0) { +// diffMedian = (sortedDiffs[(n-1)/2 - 1] + sortedDiffs[(n-1)/2]) / 2.0; +// } else { +// diffMedian = sortedDiffs[(n-1)/2]; +// } +// features[5] = diffMedian / 64.0; + +// // 7. 位宽差分极差/64 +// int minDiff = sortedDiffs[0]; +// int maxDiff = sortedDiffs[n-2]; +// features[4] = (maxDiff - minDiff) / 64.0; + + // 8. 位宽差分标准差/64 + double diffMean = diffSum / (n-1); + double diffVariance = 0; + for (int diff : diffs) { + diffVariance += Math.pow(diff - diffMean, 2); + } + diffVariance /= (n-1); + double diffStdDev = Math.sqrt(diffVariance); + features[4] = diffStdDev / 64.0; + } else { + features[3] = 0; + features[4] = 0; +// features[6] = 0; +// features[7] = 0; + } + + // 9. 位宽游程平均长度/(n/8) + double avgRunLength = 0; + int runCount = 0; + int currentRunLength = 1; + for (int i = 1; i < n; i++) { + if (bitWidths[i] == bitWidths[i-1]) { + currentRunLength++; + } else { + avgRunLength += currentRunLength; + runCount++; + currentRunLength = 1; + } + } + avgRunLength += currentRunLength; + runCount++; + avgRunLength /= runCount; + features[5] = avgRunLength / (n / 8.0); + + // 10. 位宽局部最大值密度（中间值大于左右值的个数） + int localMaxCount = 0; + for (int i = 1; i < n-1; i++) { + if (bitWidths[i] > bitWidths[i-1] && bitWidths[i] > bitWidths[i+1]) { + localMaxCount++; + } + } + features[6] = localMaxCount / (double)(n-2); + + // 11. 位宽局部最大值的平均相对幅度（每个局部最大值减相邻值的平均值） + double totalRelativeAmplitude = 0; + int localMaxWithAmplitude = 0; + for (int i = 1; i < n-1; i++) { + if (bitWidths[i] > bitWidths[i-1] && bitWidths[i] > bitWidths[i+1]) { + double leftDiff = bitWidths[i] - bitWidths[i-1]; + double rightDiff = bitWidths[i] - bitWidths[i+1]; + totalRelativeAmplitude += (leftDiff + rightDiff) / 2.0; + localMaxWithAmplitude++; + } + } + features[7] = (localMaxWithAmplitude > 0) ? (totalRelativeAmplitude / localMaxWithAmplitude) / 64.0 : 0; + + // 12. 位宽熵 + Map frequencyMap = new HashMap<>(); + for (int bw : bitWidths) { + frequencyMap.put(bw, frequencyMap.getOrDefault(bw, 0) + 1); + } + double entropy = 0; + for (int count : frequencyMap.values()) { + double probability = count / (double)n; + entropy -= probability * (Math.log(probability) / Math.log(2)); + } + // 归一化：最大熵为log2(64)=6 + features[8] = entropy / 6.0; + + // 13. 位宽单调段数量比例 + int monotonicSegments = 1; + Boolean isIncreasing = null; + for (int i = 1; i < n; i++) { + if (bitWidths[i] > bitWidths[i-1]) { + if (isIncreasing == null || !isIncreasing) { + monotonicSegments++; + isIncreasing = true; + } + } else if (bitWidths[i] < bitWidths[i-1]) { + if (isIncreasing == null || isIncreasing) { + monotonicSegments++; + isIncreasing = false; + } + } else { + // 相等时不变 + } + } + features[9] = monotonicSegments / (double)n; + +// // 14. 高位宽比例（大于32的比例） +// int highBitWidthCount = 0; +// for (int bw : bitWidths) { +// if (bw > 32) { +// highBitWidthCount++; +// } +// } +// features[13] = highBitWidthCount / (double)n; + + return features; + } + + public static void processChunkAndOutputFeatures(int[] scaledInts, CsvWriter datasetWriter, int chunkIndex, String file_name) throws IOException { + if (scaledInts.length == 0) return; + + // 计算每8个值的位宽序列 + int groupSize = 8; + int numGroups = (scaledInts.length + groupSize - 1) / groupSize; + int[] bitWidthsPerGroup = new int[numGroups]; + + for (int i = 0; i < numGroups; i++) { + int start = i * groupSize; + int end = Math.min(start + groupSize, scaledInts.length); + + // 找到当前组中的最大值 + int maxInGroup = 0; + for (int j = start; j < end; j++) { + if (scaledInts[j] > maxInGroup) { + maxInGroup = scaledInts[j]; + } + } + + // 计算最大值的位宽 + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidthsPerGroup[i] = bitWidth; + } + + // 计算10个特征 + double[] features = computeBitWidthFeatures(bitWidthsPerGroup); + + // 计算optimal pack size + int optimalPackSize = findOptimalPackSizeallV2(scaledInts); + + // 写入dataset.csv + String[] datasetRecord = new String[13]; + datasetRecord[0] = String.valueOf(file_name); + datasetRecord[1] = String.valueOf(chunkIndex); // 添加chunk索引作为第一列 + for (int f = 0; f < 10; f++) { + datasetRecord[f + 2] = String.valueOf(features[f]); + } + datasetRecord[12] = String.valueOf(optimalPackSize); + datasetWriter.writeRecord(datasetRecord); + } + + @Test + public void FeatureTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 创建dataset.csv文件用于存储特征和optimal pack size + String datasetPath = outputDirstr + "/features_and_best_p.csv"; + CsvWriter datasetWriter = new CsvWriter(datasetPath, ',', StandardCharsets.UTF_8); + + // 写入dataset.csv的表头（增加chunk_index作为第一列） + String[] datasetHeader = { + "Dataset", + "chunk_index", + "mean_bitwidth", + "median_bitwidth", +// "range_bitwidth", + "std_bitwidth", + "mean_diff_bitwidth", +// "median_diff_bitwidth", +// "range_diff_bitwidth", + "std_diff_bitwidth", + "avg_run_length", + "local_max_density", + "local_max_amplitude", + "entropy", + "monotonic_segments", +// "high_bitwidth_ratio", + "optimal_pack_size" + }; + datasetWriter.writeRecord(datasetHeader); + + // 用于跟踪chunk索引 + int globalChunkIndex = 0; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 处理chunk并输出特征到dataset.csv（只处理第一次迭代） + processChunkAndOutputFeatures(scaledInts, datasetWriter, globalChunkIndex, file.getName()); + globalChunkIndex++; + } + } + } + + // 关闭dataset.csv写入器 + datasetWriter.close(); + System.out.println("Dataset saved to: " + datasetPath); + System.out.println("Total chunks processed: " + globalChunkIndex); + } + + @Test + public void FeatureAfterSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 创建dataset.csv文件用于存储特征和optimal pack size + String datasetPath = outputDirstr + "/features_and_best_p_sprintz.csv"; + CsvWriter datasetWriter = new CsvWriter(datasetPath, ',', StandardCharsets.UTF_8); + + // 写入dataset.csv的表头（增加chunk_index作为第一列） + String[] datasetHeader = { + "Dataset", + "chunk_index", + "mean_bitwidth", + "median_bitwidth", +// "range_bitwidth", + "std_bitwidth", + "mean_diff_bitwidth", +// "median_diff_bitwidth", +// "range_diff_bitwidth", + "std_diff_bitwidth", + "avg_run_length", + "local_max_density", + "local_max_amplitude", + "entropy", + "monotonic_segments", +// "high_bitwidth_ratio", + "optimal_pack_size" + }; + datasetWriter.writeRecord(datasetHeader); + + // 用于跟踪chunk索引 + int globalChunkIndex = 0; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + int[] scaledInts = sprintz(scaledInt); + + // 处理chunk并输出特征到dataset.csv（只处理第一次迭代） + processChunkAndOutputFeatures(scaledInts, datasetWriter, globalChunkIndex,file.getName()); + globalChunkIndex++; + } + } + } + + // 关闭dataset.csv写入器 + datasetWriter.close(); + System.out.println("Dataset saved to: " + datasetPath); + System.out.println("Total chunks processed: " + globalChunkIndex); + } + + @Test + public void testPackSizeCostAnalysis() throws IOException { + System.out.println("\nPackSize Cost Analysis..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/packsize_cost_analysis"; + File outputDir = new File(outputDirStr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("Analyzing " + file.getName() + "..."); + + // 读取数据 + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 创建输出文件 + String outputFileName = file.getName().replace(".csv", "_cost.csv"); + String outputPath = outputDirStr + "/" + outputFileName; + CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8); + + // 写入表头 + String[] head = {"pack size", "value_cost" , "bitwidth_cost", "cost"}; + writer.writeRecord(head); + + // 对于每个数据块，分析不同packsize的cost + for (int chunkStart = 0; chunkStart < scaledInts_all.length; chunkStart += CHUNK_SIZE) { + int chunkEnd = Math.min(chunkStart + CHUNK_SIZE, scaledInts_all.length); + int[] chunkData = new int[chunkEnd - chunkStart]; + System.arraycopy(scaledInts_all, chunkStart, chunkData, 0, chunkData.length); + + System.out.println("Processing chunk " + (chunkStart/CHUNK_SIZE + 1) + + " of " + (int)Math.ceil(scaledInts_all.length / (double)CHUNK_SIZE)); + + // 为当前chunk计算每个packsize的cost + for (int packSize = 1; packSize <= CHUNK_SIZE; packSize++) { + if (packSize > chunkData.length) break; + + // 计算当前packsize下的总cost + long totalCost = 0; + + // 计算pack数量 + int numPacks = (chunkData.length + packSize - 1) / packSize; + int[] bitWidths = new int[numPacks]; + + // 计算全局最大位宽 + int globalMax = 0; + for (int value : chunkData) { + if (value > globalMax) { + globalMax = value; + } + } + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 计算每个pack的位宽 + for (int packIdx = 0; packIdx < numPacks; packIdx++) { + int start = packIdx * packSize; + int end = Math.min(start + packSize, chunkData.length); + int maxInPack = 0; + + for (int i = start; i < end; i++) { + if (chunkData[i] > maxInPack) { + maxInPack = chunkData[i]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInPack)); + bitWidths[packIdx] = bitWidth; + } + + // 使用与findOptimalPackSizeallV3相同的cost计算方法 + long cost = 0; + long value_cost = 0; + long bit_width_cost = 0; + for (int packIdx = 0; packIdx < numPacks; packIdx++) { + int start = packIdx * packSize; + int end = Math.min(start + packSize, chunkData.length); + int valuesInPack = end - start; + + // 找到pack中的最大位宽 + int maxBitWidth = 0; + for (int i = start; i < end; i++) { + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, chunkData[i])); + if (bitWidth > maxBitWidth) { + maxBitWidth = bitWidth; + } + } + value_cost +=(long) valuesInPack * maxBitWidth; + } + cost += value_cost; + bit_width_cost += (long) numPacks * z; + + // 加上位宽信息的存储成本 + cost += bit_width_cost; + + // 写入结果 + String[] record = { + String.valueOf(packSize), + String.valueOf(value_cost), + String.valueOf(bit_width_cost), + String.valueOf(cost) + }; + writer.writeRecord(record); + } + } + + writer.close(); + System.out.println("Cost analysis for " + file.getName() + " completed. Output: " + outputPath); + } + } + + + @Test + public void OptimalPackSizeFiltersTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_filters"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + + int[] filters_count = new int[1]; + int pack_size = testFindOptimalPackSizeallV5(scaledInts,filters_count); + + + String[] record = { + file.toString(), + "BP+RMQ+Prune", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + // 确保pack_size至少为1 + + + } + + } + writer.close(); + + } + } + + @Test + public void OptimalPackSizeFiltersSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_filters"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + + + int[] filters_count = new int[1]; + int[] scaledInts = sprintz(scaledInt); + int pack_size = testFindOptimalPackSizeallV5(scaledInts,filters_count); + + + String[] record = { + file.toString(), + "Sprintz+RMQ+Prune", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + // 确保pack_size至少为1 + + + } + + } + writer.close(); + + } + } + + @Test + public void OptimalPackSizeFiltersPlusTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_filters_plus"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + + int[] filters_count = new int[1]; + int pack_size = testFindOptimalPackSizeallV6Plus(scaledInts,filters_count); + + + String[] record = { + file.toString(), + "BP+Prune+Plus", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + // 确保pack_size至少为1 + + + } + + } + writer.close(); + + } + } + + @Test + public void OptimalPackSizeFiltersSprintzPlusTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_filters_plus"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 1; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + + + int[] filters_count = new int[1]; + int[] scaledInts = sprintz(scaledInt); + int pack_size = testFindOptimalPackSizeallV6Plus(scaledInts,filters_count); + + + String[] record = { + file.toString(), + "Sprintz+Prune+Plus", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + // 确保pack_size至少为1 + + + } + + } + writer.close(); + + } + } + + @Test + public void VaryPageSizeOptimalPackSizeFiltersPlusTest() throws IOException { + System.out.println("\nPerformance Testing with variable page size..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_filters_plus_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的不同page sizes + int[] pageSizes = {32, 64, 128, 256, 512, 1024, 2048, 4096,8192}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Page size", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 1; // 增加重复次数以获得更准确的时间测量 + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 对每个page size进行测试 + for (int pageSize : pageSizes) { + System.out.println("Testing with page size: " + pageSize); + + long totalModelCost = 0; + long totalModelTime = 0; + long totalModelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + int totalFilterCount = 0; + + for (int i = 0; i < numbers.size(); i += pageSize) { + int end = Math.min(i + pageSize, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 使用特定的page size进行测试 + int[] filters_count = new int[1]; + + long startTime = System.nanoTime(); + int pack_size = testFindOptimalPackSizeallV6Plus(scaledInts, filters_count); + + + String[] record = { + file.toString(), + String.valueOf(pageSize), + "BP+Prune+Plus", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + } + + totalModelCost += modelCost; + totalModelTime += modelTime; + totalModelDecodeTime += modelDecodeTime; + } + } + + writer.close(); + } + } + + @Test + public void VaryPageSizeOptimalPackSizeFiltersPlusSprintzTest() throws IOException { + System.out.println("\nPerformance Testing with variable page size..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_filters_plus_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的不同page sizes + int[] pageSizes = {32, 64, 128, 256, 512, 1024, 2048, 4096,8192}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Page size", + "Encoding Algorithm", + "Filter Count" + }; + writer.writeRecord(head); + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 1; // 增加重复次数以获得更准确的时间测量 + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 对每个page size进行测试 + for (int pageSize : pageSizes) { + System.out.println("Testing with page size: " + pageSize); + + long totalModelCost = 0; + long totalModelTime = 0; + long totalModelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + int totalFilterCount = 0; + + for (int i = 0; i < numbers.size(); i += pageSize) { + int end = Math.min(i + pageSize, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + // 使用特定的page size进行测试 + int[] filters_count = new int[1]; + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int pack_size = testFindOptimalPackSizeallV6Plus(scaledInts, filters_count); + + + String[] record = { + file.toString(), + String.valueOf(pageSize), + "Sprintz+Prune+Plus", + String.valueOf(filters_count[0]), + }; + writer.writeRecord(record); + } + + totalModelCost += modelCost; + totalModelTime += modelTime; + totalModelDecodeTime += modelDecodeTime; + } + } + + writer.close(); + } + } + + + @Test + public void BPTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = 8; + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + /// vary pack size + + @Test + public void OptimalPackSizeN2Test() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_N2_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 200; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV2(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void OptimalPackSizeN2SprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_N2_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV2(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void OptimalPackSizeRMQTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + if(!file.getName().equals("PM10-dust.csv")) continue; + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 500; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void OptimalPackSizeRMQSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8_sprintz"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void VaryPackSizeTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_pack_size"; + System.out.println("\nTesting Varying Pack Sizes..."); + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdir(); + + File dir = new File(directory); + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("\nTesting file: " + file.getName()); + String outputFile = outputDirStr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputFile, ',', StandardCharsets.UTF_8); + + // 更新表头 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Pack size", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + // 读取数据 + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + int time_of_repeat = 50; + + // 缩放数据 + int batchSize = 1024; + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个pack size + for (int packSize : packSizes) { + System.out.println("Testing pack size: " + packSize); + + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 编码 + long startEncodeTime = System.nanoTime(); + + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + } + } + + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; + + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + + // 写入结果 + String[] record = { + file.toString(), + "BitPacking", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(totalPoints), + String.valueOf(packSize), + String.valueOf(avgCompressedSize), + String.valueOf(compressionRatio) + }; + writer.writeRecord(record); + + System.out.println(" Pack size: " + packSize + + ", Encode throughput: " + String.format("%.2f", encodeThroughput) + " MB/s" + + ", Decode throughput: " + String.format("%.2f", decodeThroughput) + " MB/s" + + ", Compression ratio: " + String.format("%.4f", compressionRatio)); + } + + writer.close(); + System.out.println("Results saved to: " + outputFile); + } + } + + @Test + public void VaryPackSizeSprintzTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_pack_size"; + System.out.println("\nTesting Varying Pack Sizes..."); + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdir(); + + File dir = new File(directory); + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("\nTesting file: " + file.getName()); + String outputFile = outputDirStr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputFile, ',', StandardCharsets.UTF_8); + + // 更新表头 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Pack size", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + // 读取数据 + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + int time_of_repeat = 50; + + // 缩放数据 + int batchSize = 1024; + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个pack size + for (int packSize : packSizes) { + System.out.println("Testing pack size: " + packSize); + + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + // 编码 + long startEncodeTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + int[] sprintzDecoded = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + } + } + + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; + + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + + // 写入结果 + String[] record = { + file.toString(), + "Sprintz", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(totalPoints), + String.valueOf(packSize), + String.valueOf(avgCompressedSize), + String.valueOf(compressionRatio) + }; + writer.writeRecord(record); + + System.out.println(" Pack size: " + packSize + + ", Encode throughput: " + String.format("%.2f", encodeThroughput) + " MB/s" + + ", Decode throughput: " + String.format("%.2f", decodeThroughput) + " MB/s" + + ", Compression ratio: " + String.format("%.4f", compressionRatio)); + } + + writer.close(); + System.out.println("Results saved to: " + outputFile); + } + } + + @Test + public void OptimalPackSizePruneTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_only_Prune_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + int pack_size = findOptimalPackSizeallV6(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+Prune", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + @Test + public void OptimalPackSizePruneSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_only_Prune_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + int pack_size = findOptimalPackSizeallV6(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+Prune", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + @Test + public void OptimalPackSizePruneRMQTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_Prune_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 500; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + int pack_size = findOptimalPackSizeallV5(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ+Prune", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + @Test + public void OptimalPackSizePruneRMQSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_Prune_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + int pack_size = findOptimalPackSizeallV5(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ+Prune", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + @Test + public void OptimalPackSizePrunePlusTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_only_Prune_Plus_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + + int pack_size = findOptimalPackSizeallV6Plus(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+Prune+Plus", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + @Test + public void OptimalPackSizePrunePlusSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_only_Prune_Plus_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("PM10-dust.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + int pack_size = findOptimalPackSizeallV6Plus(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] sprintzdata = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+Prune+Plus", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); +// break; + } + } + + + /// vary page size + + @Test + public void TestVariablePageSizeBP() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { +// if(!file.getName().equals("Air-pressure.csv")) continue; + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); +// int remainder = scaledInts.length % pack_size; +// int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + +// System.out.println(cur_cost); +// System.out.println("pack size: "+8); + + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime.divide(BigInteger.valueOf(time_of_repeat))); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "BP", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + } +// break; + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeSprintz() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_sprintz_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { + int pack_size = (int) Math.pow(2, pack_size_exp); + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime.divide(BigInteger.valueOf(time_of_repeat))); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "Sprintz", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + } + } + writer.close(); +// break; + } + } + + + @Test + public void TestVariablePageSizeBPN2() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_page_size_N2"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if(!file.getName().equals("Food-price.csv")) continue; + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 100; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + BigDecimal modelCost = new BigDecimal("0.0"); + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); + int pack_size = findOptimalPackSizeallV2(scaledInts); +// System.out.println("pack size: "+pack_size); +// int remainder = scaledInts.length % pack_size; +// int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; + + // 创建新数组，长度补齐为pack_size的倍数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + + int cur_cost = compressedData.length * 8; +// System.out.println(cur_cost); + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost = modelCost.add(BigDecimal.valueOf(cur_cost)); + } + } + + modelCost = modelCost.divide(BigDecimal.valueOf(time_of_repeat), + 10, // 精度：保留10位小数 + RoundingMode.HALF_UP // 舍入模式：四舍五入 + ); + modelTime = (modelTime.divide(BigInteger.valueOf(time_of_repeat))); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + BigDecimal model_ratio = modelCost.divide(BigDecimal.valueOf(numbers.size()) + .multiply(BigDecimal.valueOf(64L)), + 10, // 精度：保留10位小数 + RoundingMode.HALF_UP // 舍入模式：四舍五入 + ); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "BP-All", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio.doubleValue()) + }; + writer.writeRecord(record); + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + } +// break; + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeSprintzN2() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_page_size_N2"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 10; +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int pack_size = findOptimalPackSizeallV2(scaledInts); + + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime.divide(BigInteger.valueOf(time_of_repeat))); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "Sprintz-All", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + } + } + writer.close(); +// break; + } + } + + + @Test + public void TestVariablePageSizeBPRMQ() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_page_size_RMQ"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); + int pack_size = findOptimalPackSizeallV3(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000) / (double) (modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "BPStar", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeSprintzRMQ() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_page_size_RMQ"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int pack_size = findOptimalPackSizeallV3(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + modelTime += (duration); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000) / (double) (modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "Sprintz-Star", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeBPonlyPrune() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_only_Prune_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 100; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + BigDecimal modelCost = new BigDecimal("0.0"); + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); + int pack_size = findOptimalPackSizeallV6Plus(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost = modelCost.add(BigDecimal.valueOf(cur_cost)); + } + } + + modelCost = modelCost.divide(BigDecimal.valueOf(time_of_repeat), + 10, // 精度：保留10位小数 + RoundingMode.HALF_UP // 舍入模式：四舍五入 + ); + modelTime = (modelTime.divide(BigInteger.valueOf(time_of_repeat))); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); +// double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigDecimal model_ratio = modelCost.divide(BigDecimal.valueOf(numbers.size()) + .multiply(BigDecimal.valueOf(64.0)), + 10, // 精度：保留10位小数 + RoundingMode.HALF_UP // 舍入模式：四舍五入 + ); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "BPStar", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio.doubleValue()) + }; + writer.writeRecord(record); + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + } + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeSprintzonlyPrune() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_only_Prune_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 50; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int pack_size = findOptimalPackSizeallV6Plus(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = modelTime.divide(BigInteger.valueOf(time_of_repeat)); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); +// double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); +// double modelDecodeTime_throughput = (double) (numbers.size() * 8000) / (double) (modelDecodeTime); + + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "Sprintz-Star", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeBPPruneRMQ() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_Prune_RMQ_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 10; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + +// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); + +// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) +// continue; +// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) +// .stream().max(Integer::compare).orElse(0); + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInts = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + + long startTime = System.nanoTime(); + int pack_size = findOptimalPackSizeallV8(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = modelTime.divide(BigInteger.valueOf(time_of_repeat)); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "BP-Prune-RMQ", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + + @Test + public void TestVariablePageSizeSprintzPruneRMQ() throws IOException { + System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_Prune_RMQ_vary_page_size"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + + // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) + int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); + String Output = outputDirstr+"/"+file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "m", + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", +// "Pack Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + + int time_of_repeat = 10; // 减少重复次数以加快测试速度 +// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); +// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + +// 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + // 测试每个chunk size + for (int chunkSize : chunkSizes) { + System.out.println("Testing chunk size: " + chunkSize); +// System.out.println(numbers.subList(0,1000)); + + for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { +// int pack_size = (int) Math.pow(2, pack_size_exp); + + int modelCost = 0; + BigInteger modelTime = new BigInteger("0"); + BigInteger modelDecodeTime = new BigInteger("0"); + + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += chunkSize) { + + int end = Math.min(i + chunkSize, numbers.size()); + int[] scaledInt = new int[end-i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + int pack_size = findOptimalPackSizeallV8(scaledInts); + // 计算需要的组数 + int numGroups = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * pack_size; + int endIdx = Math.min(startIdx + pack_size, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + int cur_cost = compressedData.length * 8; + long duration = System.nanoTime() - startTime; + + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime = modelDecodeTime.add(BigInteger.valueOf(decodeDuration)); + + modelTime = modelTime.add(BigInteger.valueOf(duration)); + modelCost += cur_cost; + } + } + + modelCost /= time_of_repeat; + modelTime = modelTime.divide(BigInteger.valueOf(time_of_repeat)); + modelDecodeTime = modelDecodeTime.divide(BigInteger.valueOf(time_of_repeat)); + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + BigInteger modelTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelTime); + BigInteger modelDecodeTime_throughput = BigInteger.valueOf(numbers.size()) + .multiply(BigInteger.valueOf(8000L)) + .divide(modelDecodeTime); + String[] record = { + String.valueOf(chunkSize), + file.toString(), + "Sprintz-Star", + String.valueOf(modelTime_throughput.longValue()), + String.valueOf(modelDecodeTime_throughput.longValue()), + String.valueOf(numbers.size()), + String.valueOf(modelCost), +// String.valueOf(pack_size), + String.valueOf(model_ratio) + }; + System.out.println(modelTime_throughput.longValue()); + System.out.println(modelDecodeTime_throughput.longValue()); + writer.writeRecord(record); + } + } + writer.close(); +// break; + } + } + + +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java deleted file mode 100644 index 0db2a6b9e..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/BDCTest.java +++ /dev/null @@ -1,575 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.*; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** - * BDC（dynamic bit packing）实现 - 无 Delta / 无 ZigZag 版本 - * - * 核心函数： - * - encodeDynamicPacking(int[] paddedArray, int packSize, int qmbdLen) -> byte[] - * - decodeDynamicPacking(byte[] encoded, int packSize, int originalLength) -> int[] - * - * 主要改动：bit-packing 部分改用“位缓冲（bit buffer）连续写入 / 读取”的实现， - * 与之前你给出的 pack8Values / unpack8Values 思路一致（按 width 连续写位流）。 - */ -public class BDCTest { - - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - private static final int CHUNK_SIZE = 1024; - - // -------------------- 工具 -------------------- - public static int getBitWidth(int num) { - if (num == 0) return 1; - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static void intToBytesBE(int value, byte[] dst, int pos) { - dst[pos] = (byte) (value >> 24); - dst[pos+1] = (byte) (value >> 16); - dst[pos+2] = (byte) (value >> 8); - dst[pos+3] = (byte) (value); - } - public static void packCountToBytesBE(int value, byte[] dst, int pos) { - dst[pos] = (byte) (value); - } - - public static int bytesToIntBE(byte[] src, int pos) { - int v = 0; - v |= (src[pos] & 0xFF) << 24; - v |= (src[pos+1] & 0xFF) << 16; - v |= (src[pos+2] & 0xFF) << 8; - v |= (src[pos+3] & 0xFF); - return v; - } - public static int bytesToPackCount(byte[] src, int pos) { - int v = 0; - v |= (src[pos] & 0xFF); - return v; - } - - // -------------------- 缩放 (无 Delta/ZigZag) -------------------- - private static int[] scaleNumbers(List numbers, int decimalMax) { - BigDecimal scale = BigDecimal.TEN.pow(decimalMax); - int size = numbers.size(); - int[] result = new int[size]; - if (size == 0) return result; - - BigDecimal min = null; - BigDecimal[] scaledValues = new BigDecimal[size]; - - for (int i = 0; i < size; i++) { - BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); - scaledValues[i] = val; - if (min == null || val.compareTo(min) < 0) { - min = val; - } - } - - for (int i = 0; i < size; i++) { - BigDecimal current = scaledValues[i].subtract(min); - result[i] = current.toBigInteger().intValue(); - } - return result; - } - - // -------------------- bit-packing (基于位缓冲，行为与上次 pack8/unpack8 思路一致) -------------------- - /** - * 将 values（任意长度，但通常为 packSize 的整数倍）以 width 位连续写成字节流（高位先出）。 - * 返回字节数组（末尾会按位补齐到整字节）。 - */ - private static byte[] bitPackList(ArrayList values, int width) { - if (values == null || values.isEmpty()) return new byte[0]; - - // 计算需要的字节数 - int totalBits = values.size() * width; - int totalBytes = (totalBits + 7) / 8; - byte[] encoded_result = new byte[totalBytes]; - - // 处理8的倍数个数值 - int encode_pos = bitPacking(values, 0, width, 0, encoded_result); - - // 处理剩余的值（如果不是8的倍数） - int processedCount = (values.size() / 8) * 8; - int remaining = values.size() - processedCount; - - if (remaining > 0) { - // 使用原来的位打包逻辑处理剩余的值 - long bitBuffer = 0L; - int bitCount = 0; - long mask = (width >= 63) ? -1L : ((1L << width) - 1L); - - for (int i = processedCount; i < values.size(); i++) { - int v = values.get(i); - long vv = ((long) v) & mask; - bitBuffer = (bitBuffer << width) | vv; - bitCount += width; - - while (bitCount >= 8) { - int shift = bitCount - 8; - int b = (int) ((bitBuffer >>> shift) & 0xFFL); - encoded_result[encode_pos++] = (byte) b; - bitCount -= 8; - if (bitCount == 0) { - bitBuffer = 0L; - } else { - bitBuffer = bitBuffer & ((1L << bitCount) - 1L); - } - } - } - - // 写剩余的不足一字节的位 - if (bitCount > 0) { - int b = (int) ((bitBuffer << (8 - bitCount)) & 0xFFL); - encoded_result[encode_pos++] = (byte) b; - } - } - - // 创建正确大小的结果数组 - if (encode_pos < totalBytes) { - byte[] result = new byte[encode_pos]; - System.arraycopy(encoded_result, 0, result, 0, encode_pos); - return result; - } - - return encoded_result; - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - - } - - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - - return encode_pos; - - } - - /** - * 从 encoded[pos..] 连续读取 count 个 width-bit 的值，返回 UnpackResult 包含读出的值与消耗的字节数。 - */ - private static class UnpackResult { - ArrayList values; - int bytesConsumed; - UnpackResult(ArrayList v, int c) { values = v; bytesConsumed = c; } - } - - private static UnpackResult bitUnpackToList(byte[] encoded, int pos, int width, int count) { - ArrayList out = new ArrayList<>(count); - long buffer = 0L; - int totalBits = 0; - int idx = pos; - int consumed = 0; - long mask = (width >= 63) ? -1L : ((1L << width) - 1L); - - while (out.size() < count) { - // 确保 buffer 中至少有 width 位 - while (totalBits < width) { - if (idx >= encoded.length) { - // 输入不足 —— 返回当前已解出的 - return new UnpackResult(out, consumed); - } - buffer = (buffer << 8) | (encoded[idx] & 0xFFL); - idx++; - consumed++; - totalBits += 8; - } - int shift = totalBits - width; - long val = (buffer >>> shift) & mask; - out.add((int) val); - totalBits -= width; - if (totalBits == 0) { - buffer = 0L; - } else { - buffer = buffer & ((1L << totalBits) - 1L); - } - } - return new UnpackResult(out, consumed); - } - - // -------------------- 动态分包（针对每组 group_size 个值的 group-bitwidth 序列） -------------------- - /** - * encodeDynamicPacking: - * 输出格式: - * [0] packCount (1 byte) （原来你写成 header 1 byte） - * for each pack: - * [1 byte] packBitWidth - * [1 byte] numGroups (每组包含 packSize 个值) - * [payload bytes] bit-packed (numGroups * packSize values) using packBitWidth - * - * 说明：为了和你现有调用兼容，header 使用了 1 字节 packCount（如需更大 packCount，请改为 4 字节）。 - */ - public static byte[] encodeDynamicPacking(int[] paddedArray, int packSize, int qmbdLen) { - if (paddedArray == null) return new byte[0]; - int totalGroups = paddedArray.length / packSize; - - ArrayList queueValues = new ArrayList<>(); - int maxBD = 0; - final int SUBHEADER_BITS_ESTIMATE = 40; // 5 bytes header ~= 40 bits - - List packPayloads = new ArrayList<>(); - List packNumGroups = new ArrayList<>(); - List packBitWidths = new ArrayList<>(); - - for (int g = 0; g < totalGroups; g++) { - int start = g * packSize; - int groupMax = 0; - for (int k = 0; k < packSize; k++) { - int val = paddedArray[start + k]; - if (val > groupMax) groupMax = val; - } - int groupBD = getBitWidth(groupMax); - - if (queueValues.isEmpty()) { - for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); - maxBD = groupBD; - if (queueValues.size() / packSize >= qmbdLen) { - int groupCount = queueValues.size() / packSize; - byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); - packPayloads.add(payload); - packNumGroups.add(groupCount); - packBitWidths.add(Math.max(1, maxBD)); - queueValues.clear(); - maxBD = 0; - } - continue; - } - - if (groupBD > maxBD) { - int wastedBits = (groupBD - maxBD) * queueValues.size(); - if (wastedBits >= SUBHEADER_BITS_ESTIMATE || (queueValues.size() / packSize) >= qmbdLen) { - int groupCount = queueValues.size() / packSize; - byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); - packPayloads.add(payload); - packNumGroups.add(groupCount); - packBitWidths.add(Math.max(1, maxBD)); - queueValues.clear(); - maxBD = 0; - - for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); - maxBD = groupBD; - } else { - for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); - maxBD = Math.max(maxBD, groupBD); - if ((queueValues.size() / packSize) >= qmbdLen) { - int groupCount = queueValues.size() / packSize; - byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); - packPayloads.add(payload); - packNumGroups.add(groupCount); - packBitWidths.add(Math.max(1, maxBD)); - queueValues.clear(); - maxBD = 0; - } - } - } else { - for (int k = 0; k < packSize; k++) queueValues.add(paddedArray[start + k]); - if ((queueValues.size() / packSize) >= qmbdLen) { - int groupCount = queueValues.size() / packSize; - byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); - packPayloads.add(payload); - packNumGroups.add(groupCount); - packBitWidths.add(Math.max(1, maxBD)); - queueValues.clear(); - maxBD = 0; - } - } - } - - if (!queueValues.isEmpty()) { - int groupCount = queueValues.size() / packSize; - byte[] payload = bitPackList(queueValues, Math.max(1, maxBD)); - packPayloads.add(payload); - packNumGroups.add(groupCount); - packBitWidths.add(Math.max(1, maxBD)); - queueValues.clear(); - maxBD = 0; - } - -// // 写出最终 byte[] -// ByteArrayOutputStream out = new ByteArrayOutputStream(); -// int packCount = packPayloads.size(); -// // 1 byte packCount -// out.write(packCount & 0xFF); -// try { -// for (int i = 0; i < packCount; i++) { -// int bw = packBitWidths.get(i); -// int ng = packNumGroups.get(i); -// out.write((byte) (bw & 0xFF)); -// out.write((byte) (ng & 0xFF)); -// out.write(packPayloads.get(i)); -// } -// } catch (IOException e) { -// e.printStackTrace(); -// } -// byte[] result = new byte[out.toByteArray().length]; -// for (int i = 0; i < result.length; i++) { -// result[i] = out.toByteArray()[i]; -// } -// return result; - int packCount = packPayloads.size(); - -// 先计算总长度：1 byte packCount + for each pack (1 byte bw + 1 byte numGroups + payload length) - int totalLen = 1; // packCount - for (int i = 0; i < packCount; i++) { - totalLen += 1; // bit width - totalLen += 1; // numGroups - totalLen += packPayloads.get(i).length; // payload - } - -// 分配数组并填充 - byte[] result = new byte[totalLen]; - int pos = 0; - result[pos++] = (byte) (packCount & 0xFF); - - for (int i = 0; i < packCount; i++) { - int bw = packBitWidths.get(i); - int ng = packNumGroups.get(i); - byte[] payload = packPayloads.get(i); - - result[pos++] = (byte) (bw & 0xFF); - result[pos++] = (byte) (ng & 0xFF); -// for (byte b : payload) { -// result[pos++] = (byte) (b & 0xFF); -// } - for (int bi = 0; bi < payload.length; bi++) { - result[pos + bi] = payload[bi]; - } -// System.arraycopy(payload, 0, result, pos, payload.length); - pos += payload.length; - } - -// 返回结果 - return result; - } - - // decode 对应的 encodeDynamicPacking 格式 - public static int[] decodeDynamicPacking(byte[] encoded, int packSize, int originalLength) { - if (encoded == null || encoded.length < 1) return new int[0]; - int pos = 0; - int packCount = bytesToPackCount(encoded, pos); pos += 1; - - ArrayList all = new ArrayList<>(originalLength); - for (int p = 0; p < packCount; p++) { - if (pos >= encoded.length) break; - int bw = encoded[pos++] & 0xFF; - int numGroups = bytesToPackCount(encoded, pos); pos += 1; - int numValues = numGroups * packSize; - UnpackResult ur = bitUnpackToList(encoded, pos, bw, numValues); - all.addAll(ur.values); - pos += ur.bytesConsumed; - } - int[] out = new int[originalLength]; - for (int i = 0; i < originalLength && i < all.size(); i++) out[i] = all.get(i); - if (all.size() < originalLength) { - for (int i = all.size(); i < originalLength; i++) out[i] = 0; - } - return out; - } - - // -------------------- 主测试（参考你原 main） -------------------- - public static void main(String[] args) throws IOException { - System.out.println("\nPerformance Testing (Dynamic pack over 8-values groups)..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BDC"; - File outputDir = new File(outputDirstr); - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - if (!dir.exists()) { - System.err.println("Directory not found: " + directory); - return; - } - - final int groupSize = 8; // 每8个值一个 group - final int qmbdLen = 8; // QMBD 队列上限（可调） - - for (File file : Objects.requireNonNull(dir.listFiles())) { - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr + "/" + file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - csvReader.close(); - - int time_of_repeat = 50; - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - long startTime = System.nanoTime(); - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - - // pad to multiple of groupSize - int remainder = scaledInts.length % groupSize; - int paddingLength = (remainder == 0) ? 0 : groupSize - remainder; - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); -// int groups = paddedArray.length / groupSize; -// int[] bitWidths = new int[groups]; -// int gidx = 0; -// for (int si = 0; si < paddedArray.length; si += groupSize) { -// long maxInGroup = 0; -// for (int sj = si; sj < si + groupSize; ++sj) { -// long v = paddedArray[sj]; -// if (v > maxInGroup) maxInGroup = v; -// } -// int bitWidth = 0; -// if (maxInGroup > 0) { -// bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); -// } else { -// bitWidth = 0; -// } -// bitWidths[gidx++] = bitWidth; -// } - // encode using dynamic packing over groups of size 8 - byte[] compressed = encodeDynamicPacking(paddedArray, groupSize, qmbdLen); - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += (long) compressed.length * 8L; - - // decode time - long startDec = System.nanoTime(); - int[] decoded = decodeDynamicPacking(compressed, groupSize,paddedArray.length); - long decDur = System.nanoTime() - startDec; - modelDecodeTime += decDur; - -// // 验证（只在第一次迭代验证原始 scaledInts 部分） -// if (j == 0) { -// boolean ok = true; -// for (int k = 0; k < scaledInts.length; k++) { -// if (scaledInts[k] != decoded[k]) { -// ok = false; -// System.err.println("Mismatch at idx " + k + ": expect " + scaledInts[k] + " got " + decoded[k]); -// break; -// } -// } -// if (ok) System.out.println("Chunk decoded OK."); -// } - } - } - - modelCost = modelCost / time_of_repeat; - modelTime = modelTime / time_of_repeat; - modelDecodeTime = modelDecodeTime / time_of_repeat; - - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime == 0 ? 1 : modelTime); - double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime == 0 ? 1 : modelDecodeTime); - - String[] record = { - file.toString(), - "BP_dynamic", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); - - System.out.println("Encoding throughput: " + modelTime_throughput + " MB/s"); - System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); - System.out.println("Compression ratio: " + model_ratio); - } - } - - @Test - public void dummyTest() { - // placeholder for IDE/test runner - } -} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java deleted file mode 100644 index 853b0603a..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPacking.java +++ /dev/null @@ -1,843 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class DPOctadPacking { - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - private static final int CHUNK_SIZE = 1024; - - static String trimStr(String s) { - if (s == null) return ""; - int a = 0; - while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; - if (a == s.length()) return ""; - int b = s.length() - 1; - while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; - return s.substring(a, b + 1); - } - - static String stripEnclosingQuotes(String s) { - if (s == null) return ""; - if (s.length() >= 2) { - char f = s.charAt(0); - char l = s.charAt(s.length() - 1); - if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { - return s.substring(1, s.length() - 1); - } - } - return s; - } - - // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping - static long[] scaleNumbers(List numbers, int decimalMax) { - int n = numbers.size(); - long[] result = new long[n]; - if (n == 0) return result; - - BigDecimal scale = BigDecimal.ONE; - for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); - - BigDecimal[] vals = new BigDecimal[n]; - for (int i = 0; i < n; ++i) { - String s = trimStr(numbers.get(i)); - s = stripEnclosingQuotes(s); - if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } - s = s.replace(",", ""); // remove thousands sep - - // If scientific notation present, BigDecimal can parse it - try { - BigDecimal bd = new BigDecimal(s); - BigDecimal scaled = bd.multiply(scale); - // rounding to nearest whole - BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); - vals[i] = rounded; - } catch (Exception ex) { - // fallback: parse double - try { - double dv = Double.parseDouble(s); - BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); - vals[i] = bd.setScale(0, RoundingMode.HALF_UP); - } catch (Exception ex2) { - System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); - vals[i] = BigDecimal.ZERO; - } - } - } - - // find min - BigDecimal minv = vals[0]; - for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; - - for (int i = 0; i < n; ++i) { - BigDecimal shifted = vals[i].subtract(minv); - // clamp to long range - try { - BigInteger bi = shifted.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (ArithmeticException ae) { - // not an integer exactly: fallback by converting to long with rounding - BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); - try { - BigInteger bi = rounded.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (Exception ex) { - result[i] = 0; - } - } - } - return result; - } - - - // 动态规划结果类 - static class PackingPlan { - int optimalC; - List groupSizes; // 每个分组包含的块数 - List groupBitWidths; // 每个分组的位宽 - int totalCost; - - PackingPlan(int optimalC, List groupSizes, List groupBitWidths, int totalCost) { - this.optimalC = optimalC; - this.groupSizes = groupSizes; - this.groupBitWidths = groupBitWidths; - this.totalCost = totalCost; - } - } - - // 计算最优分组方案的完整实现 - public static PackingPlan computeOptimalPackingPlan(int[] bitWidths, int pack_size) { - int N = bitWidths.length; - if (N == 0) { - return new PackingPlan(0, new ArrayList<>(), new ArrayList<>(), 0); - } - - // 预计算所有区间的最大位宽 - int[][] maxB = new int[N][N]; - for (int l = 0; l < N; l++) { - maxB[l][l] = bitWidths[l]; - for (int r = l + 1; r < N; r++) { - maxB[l][r] = Math.max(maxB[l][r - 1], bitWidths[r]); - } - } - - int minTotalCost = Integer.MAX_VALUE; - int bestC = 1; - List bestGroupSizes = new ArrayList<>(); - List bestGroupBitWidths = new ArrayList<>(); - - int maxPossibleC = 64 - Long.numberOfLeadingZeros(N); - - for (int C = 1; C <= maxPossibleC; C++) { - int low_C = (C == 1) ? 1 : (1 << (C - 1)); - int high_C = Math.min((1 << C) - 1, N); - - // DP表和相关记录数组 - int[][] dp = new int[N + 1][2]; - int[][] prevState = new int[N + 1][2]; - int[][] prevFlag = new int[N + 1][2]; - int[][] packSize = new int[N + 1][2]; - - // 初始化 - for (int i = 0; i <= N; i++) { - dp[i][0] = Integer.MAX_VALUE / 2; - dp[i][1] = Integer.MAX_VALUE / 2; - } - dp[0][0] = 0; - - // 动态规划计算 - for (int i = 1; i <= N; i++) { - for (int k = Math.max(1, i - high_C + 1); k <= i; k++) { - int packLength = i - k + 1; - int currentMaxB = maxB[k - 1][i - 1]; - int packCost = pack_size * packLength * currentMaxB + 6 + C; - - if (packLength < low_C) { - // 小分组，不能改变状态 - if (dp[k - 1][0] + packCost < dp[i][0]) { - dp[i][0] = dp[k - 1][0] + packCost; - prevState[i][0] = k - 1; - prevFlag[i][0] = 0; - packSize[i][0] = packLength; - } - if (dp[k - 1][1] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][1] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 1; - packSize[i][1] = packLength; - } - } else { - // 大分组，可以改变状态到1 - if (dp[k - 1][0] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][0] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 0; - packSize[i][1] = packLength; - } - if (dp[k - 1][1] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][1] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 1; - packSize[i][1] = packLength; - } - } - } - } - - // 回溯构建分组方案 - if (dp[N][1] < minTotalCost) { - minTotalCost = dp[N][1]; - bestC = C; - bestGroupSizes = new ArrayList<>(); - bestGroupBitWidths = new ArrayList<>(); - - // 回溯路径 - int i = N; - int flag = 1; - while (i > 0) { - int size = packSize[i][flag]; - int start = i - size; - int bitWidth = maxB[start][i - 1]; - - bestGroupSizes.add(0, size); - bestGroupBitWidths.add(0, bitWidth); - - i = prevState[i][flag]; - flag = prevFlag[i + size][flag]; // 注意这里要调整索引 - } - } - } - - return new PackingPlan(bestC, bestGroupSizes, bestGroupBitWidths, minTotalCost); - } - - // 计算压缩后数据总大小 - private static int calculateTotalCompressedSize(PackingPlan plan, int pack_size) { - int totalSize = 0; - - // 元数据头大小: C(5bits) + 分组数量(32bits) - totalSize += 6; // C占用5bits - totalSize += 8; // 分组数量占用32bits - - // 每个分组的元数据: packsize(plan.optimalC bits) + 位宽(5bits) - int groupMetadataBits = plan.groupSizes.size() * (plan.optimalC + 6); - totalSize += groupMetadataBits; - - // 压缩数据大小 - for (int i = 0; i < plan.groupSizes.size(); i++) { - int groupBlocks = plan.groupSizes.get(i); - int bitWidth = plan.groupBitWidths.get(i); - int dataBits = groupBlocks * pack_size * bitWidth; - totalSize += dataBits; - } - - // 转换为字节数（向上取整） - return (totalSize + 7) / 8; - } - - // 修改后的bitPacking方法，支持指定的起始位位置 - public static int bitPacking(ArrayList numbers, int start, int bit_width, - int encode_pos, byte[] encoded_result, int startBitPos) { - int block_num = (numbers.size() - start) / 8; - int currentBytePos = encode_pos; - int currentBitPos = startBitPos; - - for (int i = 0; i < block_num; i++) { - // 对每个8值块进行压缩 - for (int j = 0; j < 8; j++) { - int value = numbers.get(start + i * 8 + j); - - // 按位写入 - for (int bit = bit_width - 1; bit >= 0; bit--) { - int currentBit = (value >> bit) & 1; - encoded_result[currentBytePos] |= (currentBit << (7 - currentBitPos)); - currentBitPos++; - - if (currentBitPos == 8) { - currentBytePos++; - currentBitPos = 0; - } - } - } - } - - return currentBytePos; - } - - // 辅助：位写入器 - static class BitWriter { - final byte[] data; - int bytePos = 0; - int bitPos = 0; // 0..7, next bit to write in current byte (7 - bitPos is mask shift) - - BitWriter(byte[] data, int startBytePos) { this.data = data; this.bytePos = startBytePos; } - - // 写 numBits 位（numBits <= 64），value 的低 numBits 位被写出（big-endian bit order） - void writeBits(long value, int numBits) { - for (int i = numBits - 1; i >= 0; --i) { - int bit = (int)((value >> i) & 1L); - data[bytePos] |= (byte)(bit << (7 - bitPos)); - bitPos++; - if (bitPos == 8) { - bitPos = 0; - bytePos++; - } - } - } - - int getBytePos() { return bytePos; } - int getBitPos() { return bitPos; } - // 返回写入结束后占用的字节数（包括部分字节） - int bytesWritten() { - return bytePos + (bitPos > 0 ? 1 : 0); - } - } - - // 辅助：位读取器 - static class BitReader { - final byte[] data; - int bytePos = 0; - int bitPos = 0; - - BitReader(byte[] data, int startBytePos, int startBitPos) { - this.data = data; - this.bytePos = startBytePos; - this.bitPos = startBitPos; - } - - // 读 numBits 位并作为 long 返回（numBits <= 64） - long readBits(int numBits) { - long res = 0L; - for (int i = 0; i < numBits; ++i) { - int bit = (data[bytePos] >> (7 - bitPos)) & 1; - res = (res << 1) | bit; - bitPos++; - if (bitPos == 8) { - bitPos = 0; - bytePos++; - } - } - return res; - } - - int getBytePos() { return bytePos; } - int getBitPos() { return bitPos; } - } - - // ---- 用新的 BitWriter/BitReader 来实现压缩/解压 ---- - public static byte[] compressWithOptimalPacking(long[] paddedArray, int[] bitWidths, int pack_size, int[] encodePos) { - PackingPlan optimalPlan = computeOptimalPackingPlan(bitWidths, pack_size); - int totalSize = calculateTotalCompressedSize(optimalPlan, pack_size); // 以字节计 - byte[] compressedData = new byte[totalSize + 8]; // 预留一点空间以防计算差异（安全余量） - BitWriter writer = new BitWriter(compressedData, 0); - - // 写 C (6 bits) 和 groupCount(8 bits) - writer.writeBits(optimalPlan.optimalC, 6); - writer.writeBits(optimalPlan.groupSizes.size(), 8); - - int dataIndex = 0; - for (int gi = 0; gi < optimalPlan.groupSizes.size(); ++gi) { - int groupBlocks = optimalPlan.groupSizes.get(gi); - int bitWidth = optimalPlan.groupBitWidths.get(gi); - writer.writeBits(groupBlocks, optimalPlan.optimalC); - writer.writeBits(bitWidth, 6); - - int groupDataCount = groupBlocks * pack_size; - for (int i = 0; i < groupDataCount; ++i) { - long v = paddedArray[dataIndex++]; - long mask = (bitWidth == 64) ? ~0L : ((1L << bitWidth) - 1L); - writer.writeBits(v & mask, bitWidth); - } - } - - int finalBytes = writer.bytesWritten(); - if (finalBytes > totalSize) { - // 警告：计算函数可能低估了实际大小。这里扩容并返回实际大小。 - System.err.println(String.format("Warning: calculateTotalCompressedSize underestimated bytes: estimated=%d actual=%d. Returning actual length.", - totalSize, finalBytes)); - byte[] out = new byte[finalBytes]; - System.arraycopy(compressedData, 0, out, 0, finalBytes); - encodePos[0] = finalBytes; - return out; - } else { - // 返回精确长度拷贝 - byte[] out = new byte[finalBytes]; - System.arraycopy(compressedData, 0, out, 0, finalBytes); - encodePos[0] = finalBytes; - return out; - } - } - - public static long[] decompressWithOptimalPacking(byte[] compressedData, int originalLength, int pack_size) { - BitReader reader = new BitReader(compressedData, 0, 0); - int C = (int) reader.readBits(6); - int groupCount = (int) reader.readBits(8); - - // sanity checks - if (C <= 0 || C > 64) throw new IllegalArgumentException("Invalid C read from header: " + C); - if (groupCount < 0 || groupCount > (1 << 20)) throw new IllegalArgumentException("Suspicious groupCount: " + groupCount); - - long[] result = new long[originalLength]; - int resultIndex = 0; - - for (int gi = 0; gi < groupCount; ++gi) { - // 先确保还能读 group header - long remainingBitsBeforeGroup = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); - if (remainingBitsBeforeGroup < C + 6) { - throw new IllegalArgumentException(String.format( - "Not enough bits for group header #%d: need %d but only %d remain (bytePos=%d bitPos=%d totalBytes=%d).", - gi, C + 6, remainingBitsBeforeGroup, reader.getBytePos(), reader.getBitPos(), compressedData.length)); - } - - int groupBlocks = (int) reader.readBits(C); - int bitWidth = (int) reader.readBits(6); - if (bitWidth < 0 || bitWidth > 64) throw new IllegalArgumentException("Invalid bitWidth: " + bitWidth); - - long groupDataCount = (long) groupBlocks * (long) pack_size; - if (groupDataCount < 0 || groupDataCount > (1L << 31)) { - throw new IllegalArgumentException("Suspicious groupDataCount: " + groupDataCount); - } - - long neededDataBits = groupDataCount * (long) bitWidth; - long remainingBitsAfterHeader = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); - if (neededDataBits > remainingBitsAfterHeader) { -// continue; - throw new IllegalArgumentException(String.format( - "Not enough bits for group #%d data: need %d bits but only %d available (bytePos=%d bitPos=%d).", - gi, neededDataBits, remainingBitsAfterHeader, reader.getBytePos(), reader.getBitPos())); - } - - for (long i = 0; i < groupDataCount; ++i) { - long v = reader.readBits(bitWidth); - if (resultIndex < originalLength) result[resultIndex++] = v; - } - } - - return result; - } - - - public static void main(String[] args) throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - // 方法：强化学习 - int modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - // 1. 找出当前8个元素中的最大值 - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / 8] = bitWidth; - } - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, 8,encodePos); - int cur_cost = res.length*8; - long duration = System.nanoTime() - startTime; - - long startDecodeTime = System.nanoTime(); - decompressWithOptimalPacking(res, paddedArray.length, 8); - long endDecodeTime = System.nanoTime(); - long decodeDuration = endDecodeTime - startDecodeTime; - - modelDecodeTime += decodeDuration; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /=time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - modelDecodeTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); - double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); - String[] record = { - file.toString(), - "BP-DP", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); - } - - } - - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP_vary_pack_size"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 10; - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - - int modelCost = 0; - long modelTime = 0; - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, pack_size,encodePos); - int cur_cost = encodePos[0]*8; - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - String[] record = { - file.toString(), - "BP-DP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - writer.close(); - } - - } - - // 新增方法：测试不同chunk size的表现 - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPDP_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 -// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); -// long[] scaledInts_all = scaleNumbers(numbers, decimalMax); - - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - long[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - long[] scaledInts_all = new long[totalLength]; - - int currentIndex = 0; - for (long[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); -// -// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - int end = Math.min(i + chunkSize, numbers.size()); - long[] scaledInts = new long[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为pack_size的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking(paddedArray, bitWidths, pack_size, encodePos); - int cur_cost = encodePos[0] * 8; - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "BP-DP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); - } - } -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java deleted file mode 100644 index fc849696c..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/DPOctadPackingSprintz.java +++ /dev/null @@ -1,941 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -//import org.openjdk.jol.info.ClassLayout; -//import org.openjdk.jol.info.GraphLayout; -import java.math.BigInteger; -public class DPOctadPackingSprintz { - private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv","Basel-wind.csv", - "POI-lon.csv","Air-sensor.csv","Basel-temp.csv"); - - private static final int CHUNK_SIZE = 1024; - - - static String trimStr(String s) { - if (s == null) return ""; - int a = 0; - while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; - if (a == s.length()) return ""; - int b = s.length() - 1; - while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; - return s.substring(a, b + 1); - } - - static String stripEnclosingQuotes(String s) { - if (s == null) return ""; - if (s.length() >= 2) { - char f = s.charAt(0); - char l = s.charAt(s.length() - 1); - if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { - return s.substring(1, s.length() - 1); - } - } - return s; - } - - - static long[] scaleNumbers(List numbers, int decimalMax) { - int n = numbers.size(); - long[] result = new long[n]; - if (n == 0) return result; - - BigDecimal scale = BigDecimal.ONE; - for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); - - BigDecimal[] vals = new BigDecimal[n]; - for (int i = 0; i < n; ++i) { - String s = trimStr(numbers.get(i)); - s = stripEnclosingQuotes(s); - if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } - s = s.replace(",", ""); // remove thousands sep - - // If scientific notation present, BigDecimal can parse it - try { - BigDecimal bd = new BigDecimal(s); - BigDecimal scaled = bd.multiply(scale); - // rounding to nearest whole - BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); - vals[i] = rounded; - } catch (Exception ex) { - // fallback: parse double - try { - double dv = Double.parseDouble(s); - BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); - vals[i] = bd.setScale(0, RoundingMode.HALF_UP); - } catch (Exception ex2) { - System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); - vals[i] = BigDecimal.ZERO; - } - } - } - - // find min - BigDecimal minv = vals[0]; - for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; - - for (int i = 0; i < n; ++i) { - BigDecimal shifted = vals[i].subtract(minv); - // clamp to long range - try { - BigInteger bi = shifted.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (ArithmeticException ae) { - // not an integer exactly: fallback by converting to long with rounding - BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); - try { - BigInteger bi = rounded.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (Exception ex) { - result[i] = 0; - } - } - } - return result; - } - - public static long[] sprintz(long[] numbers) { - int size = numbers.length; - long[] result = new long[size]; - - if (size == 0) return result; // 空数组直接返回 - - long first = numbers[0]; - result[0] = first; - - long prev = first; - for (int i = 1; i < size; i++) { - long current = numbers[i]; - long diff = current - prev; - // ZigZag 编码: 正数 -> 偶数, 负数 -> 奇数 - result[i] = (diff << 1) ^ (diff >> 63); - prev = current; - } - - return result; - } - - // 动态规划结果类 - static class PackingPlan { - int optimalC; - List groupSizes; // 每个分组包含的块数 - List groupBitWidths; // 每个分组的位宽 - int totalCost; - - PackingPlan(int optimalC, List groupSizes, List groupBitWidths, int totalCost) { - this.optimalC = optimalC; - this.groupSizes = groupSizes; - this.groupBitWidths = groupBitWidths; - this.totalCost = totalCost; - } - } - - // 计算最优分组方案的完整实现 - public static PackingPlan computeOptimalPackingPlan(int[] bitWidths, int pack_size) { - int N = bitWidths.length; - if (N == 0) { - return new PackingPlan(0, new ArrayList<>(), new ArrayList<>(), 0); - } - - // 预计算所有区间的最大位宽 - int[][] maxB = new int[N][N]; - for (int l = 0; l < N; l++) { - maxB[l][l] = bitWidths[l]; - for (int r = l + 1; r < N; r++) { - maxB[l][r] = Math.max(maxB[l][r - 1], bitWidths[r]); - } - } - - int minTotalCost = Integer.MAX_VALUE; - int bestC = 1; - List bestGroupSizes = new ArrayList<>(); - List bestGroupBitWidths = new ArrayList<>(); - - int maxPossibleC = 64 - Long.numberOfLeadingZeros(N); - - for (int C = 1; C <= maxPossibleC; C++) { - int low_C = (C == 1) ? 1 : (1 << (C - 1)); - int high_C = Math.min((1 << C) - 1, N); - - // DP表和相关记录数组 - int[][] dp = new int[N + 1][2]; - int[][] prevState = new int[N + 1][2]; - int[][] prevFlag = new int[N + 1][2]; - int[][] packSize = new int[N + 1][2]; - - // 初始化 - for (int i = 0; i <= N; i++) { - dp[i][0] = Integer.MAX_VALUE / 2; - dp[i][1] = Integer.MAX_VALUE / 2; - } - dp[0][0] = 0; - - // 动态规划计算 - for (int i = 1; i <= N; i++) { - for (int k = Math.max(1, i - high_C + 1); k <= i; k++) { - int packLength = i - k + 1; - int currentMaxB = maxB[k - 1][i - 1]; - int packCost = pack_size * packLength * currentMaxB + 6 + C; - - if (packLength < low_C) { - // 小分组，不能改变状态 - if (dp[k - 1][0] + packCost < dp[i][0]) { - dp[i][0] = dp[k - 1][0] + packCost; - prevState[i][0] = k - 1; - prevFlag[i][0] = 0; - packSize[i][0] = packLength; - } - if (dp[k - 1][1] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][1] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 1; - packSize[i][1] = packLength; - } - } else { - // 大分组，可以改变状态到1 - if (dp[k - 1][0] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][0] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 0; - packSize[i][1] = packLength; - } - if (dp[k - 1][1] + packCost < dp[i][1]) { - dp[i][1] = dp[k - 1][1] + packCost; - prevState[i][1] = k - 1; - prevFlag[i][1] = 1; - packSize[i][1] = packLength; - } - } - } - } - - // 回溯构建分组方案 - if (dp[N][1] < minTotalCost) { - minTotalCost = dp[N][1]; - bestC = C; - bestGroupSizes = new ArrayList<>(); - bestGroupBitWidths = new ArrayList<>(); - - // 回溯路径 - int i = N; - int flag = 1; - while (i > 0) { - int size = packSize[i][flag]; - int start = i - size; - int bitWidth = maxB[start][i - 1]; - - bestGroupSizes.add(0, size); - bestGroupBitWidths.add(0, bitWidth); - - i = prevState[i][flag]; - flag = prevFlag[i + size][flag]; // 注意这里要调整索引 - } - } - } - - return new PackingPlan(bestC, bestGroupSizes, bestGroupBitWidths, minTotalCost); - } - - // 计算压缩后数据总大小 - private static int calculateTotalCompressedSize(PackingPlan plan, int pack_size) { - int totalSize = 0; - - // 元数据头大小: C(5bits) + 分组数量(32bits) - totalSize += 6; // C占用5bits - totalSize += 8; // 分组数量占用32bits - - // 每个分组的元数据: packsize(plan.optimalC bits) + 位宽(5bits) - int groupMetadataBits = plan.groupSizes.size() * (plan.optimalC + 6); - totalSize += groupMetadataBits; - - // 压缩数据大小 - for (int i = 0; i < plan.groupSizes.size(); i++) { - int groupBlocks = plan.groupSizes.get(i); - int bitWidth = plan.groupBitWidths.get(i); - int dataBits = groupBlocks * pack_size * bitWidth; - totalSize += dataBits; - } - - // 转换为字节数（向上取整） - return (totalSize + 7) / 8; - } - - - // 修改后的bitPacking方法，支持指定的起始位位置 - public static int bitPacking(ArrayList numbers, int start, int bit_width, - int encode_pos, byte[] encoded_result, int startBitPos) { - int block_num = (numbers.size() - start) / 8; - int currentBytePos = encode_pos; - int currentBitPos = startBitPos; - - for (int i = 0; i < block_num; i++) { - // 对每个8值块进行压缩 - for (int j = 0; j < 8; j++) { - int value = numbers.get(start + i * 8 + j); - - // 按位写入 - for (int bit = bit_width - 1; bit >= 0; bit--) { - int currentBit = (value >> bit) & 1; - encoded_result[currentBytePos] |= (currentBit << (7 - currentBitPos)); - currentBitPos++; - - if (currentBitPos == 8) { - currentBytePos++; - currentBitPos = 0; - } - } - } - } - - return currentBytePos; - } - - // 辅助：位写入器 - private static class BitWriter { - final byte[] data; - int bytePos = 0; - int bitPos = 0; // 0..7, next bit to write in current byte (7 - bitPos is mask shift) - - BitWriter(byte[] data, int startBytePos) { this.data = data; this.bytePos = startBytePos; } - - // 写 numBits 位（numBits <= 64），value 的低 numBits 位被写出（big-endian bit order） - void writeBits(long value, int numBits) { - for (int i = numBits - 1; i >= 0; --i) { - int bit = (int)((value >> i) & 1L); - data[bytePos] |= (byte)(bit << (7 - bitPos)); - bitPos++; - if (bitPos == 8) { - bitPos = 0; - bytePos++; - } - } - } - - int getBytePos() { return bytePos; } - int getBitPos() { return bitPos; } - // 返回写入结束后占用的字节数（包括部分字节） - int bytesWritten() { - return bytePos + (bitPos > 0 ? 1 : 0); - } - } - - static class BitReader { - final byte[] data; - int bytePos = 0; - int bitPos = 0; - - BitReader(byte[] data, int startBytePos, int startBitPos) { - this.data = data; - this.bytePos = startBytePos; - this.bitPos = startBitPos; - } - - // 读 numBits 位并作为 long 返回（numBits <= 64） - long readBits(int numBits) { - long res = 0L; - for (int i = 0; i < numBits; ++i) { - int bit = (data[bytePos] >> (7 - bitPos)) & 1; - res = (res << 1) | bit; - bitPos++; - if (bitPos == 8) { - bitPos = 0; - bytePos++; - } - } - return res; - } - - int getBytePos() { return bytePos; } - int getBitPos() { return bitPos; } - } - - // ---- 用新的 BitWriter/BitReader 来实现压缩/解压 ---- - public static byte[] compressWithOptimalPacking(long[] paddedArray, int[] bitWidths, int pack_size, int[] encodePos) { - PackingPlan optimalPlan = computeOptimalPackingPlan(bitWidths, pack_size); - int totalSize = calculateTotalCompressedSize(optimalPlan, pack_size); // 以字节计 - byte[] compressedData = new byte[totalSize + 8]; // 预留一点空间以防计算差异（安全余量） - BitWriter writer = new BitWriter(compressedData, 0); - - // 写 C (6 bits) 和 groupCount(8 bits) - writer.writeBits(optimalPlan.optimalC, 6); - writer.writeBits(optimalPlan.groupSizes.size(), 8); - - int dataIndex = 0; - for (int gi = 0; gi < optimalPlan.groupSizes.size(); ++gi) { - int groupBlocks = optimalPlan.groupSizes.get(gi); - int bitWidth = optimalPlan.groupBitWidths.get(gi); - writer.writeBits(groupBlocks, optimalPlan.optimalC); - writer.writeBits(bitWidth, 6); - - int groupDataCount = groupBlocks * pack_size; - for (int i = 0; i < groupDataCount; ++i) { - long v = paddedArray[dataIndex++]; - long mask = (bitWidth == 64) ? ~0L : ((1L << bitWidth) - 1L); - writer.writeBits(v & mask, bitWidth); - } - } - - int finalBytes = writer.bytesWritten(); - if (finalBytes > totalSize) { - // 警告：计算函数可能低估了实际大小。这里扩容并返回实际大小。 - System.err.println(String.format("Warning: calculateTotalCompressedSize underestimated bytes: estimated=%d actual=%d. Returning actual length.", - totalSize, finalBytes)); - byte[] out = new byte[finalBytes]; - System.arraycopy(compressedData, 0, out, 0, finalBytes); - encodePos[0] = finalBytes; - return out; - } else { - // 返回精确长度拷贝 - byte[] out = new byte[finalBytes]; - System.arraycopy(compressedData, 0, out, 0, finalBytes); - encodePos[0] = finalBytes; - return out; - } - } - - // 解压缩方法 - public static long[] decompressWithOptimalPacking(byte[] compressedData, int originalLength, int pack_size) { - BitReader reader = new BitReader(compressedData, 0, 0); - int C = (int) reader.readBits(6); - int groupCount = (int) reader.readBits(8); - - // sanity checks - if (C <= 0 || C > 64) throw new IllegalArgumentException("Invalid C read from header: " + C); - if (groupCount < 0 || groupCount > (1 << 20)) throw new IllegalArgumentException("Suspicious groupCount: " + groupCount); - - long[] result = new long[originalLength]; - int resultIndex = 0; - - for (int gi = 0; gi < groupCount; ++gi) { - // 先确保还能读 group header - long remainingBitsBeforeGroup = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); - if (remainingBitsBeforeGroup < C + 6) { - throw new IllegalArgumentException(String.format( - "Not enough bits for group header #%d: need %d but only %d remain (bytePos=%d bitPos=%d totalBytes=%d).", - gi, C + 6, remainingBitsBeforeGroup, reader.getBytePos(), reader.getBitPos(), compressedData.length)); - } - - int groupBlocks = (int) reader.readBits(C); - int bitWidth = (int) reader.readBits(6); - if (bitWidth < 0 || bitWidth > 64) throw new IllegalArgumentException("Invalid bitWidth: " + bitWidth); - - long groupDataCount = (long) groupBlocks * (long) pack_size; - if (groupDataCount < 0 || groupDataCount > (1L << 31)) { - throw new IllegalArgumentException("Suspicious groupDataCount: " + groupDataCount); - } - - long neededDataBits = groupDataCount * (long) bitWidth; - long remainingBitsAfterHeader = ((long)(compressedData.length - reader.getBytePos())) * 8L - reader.getBitPos(); - if (neededDataBits > remainingBitsAfterHeader) { -// continue; - throw new IllegalArgumentException(String.format( - "Not enough bits for group #%d data: need %d bits but only %d available (bytePos=%d bitPos=%d).", - gi, neededDataBits, remainingBitsAfterHeader, reader.getBytePos(), reader.getBitPos())); - } - - for (long i = 0; i < groupDataCount; ++i) { - long v = reader.readBits(bitWidth); - if (resultIndex < originalLength) result[resultIndex++] = v; - } - } - - return result; - } - - - // 读取指定位数的辅助函数 - private static int readBits(byte[] data, int bytePos, int bitPos, int numBits) { - int result = 0; - for (int i = 0; i < numBits; i++) { - int bit = (data[bytePos] >> (7 - bitPos)) & 1; - result = (result << 1) | bit; - bitPos++; - if (bitPos == 8) { - bytePos++; - bitPos = 0; - } - } - return result; - } - - public static long[] sprintzDecode(long[] encodedData) { - int size = encodedData.length; - long[] result = new long[size]; - - if (size == 0) return result; - - // 第一个元素是原始值 - result[0] = encodedData[0]; - - // 后续元素需要ZigZag解码和累加 - long prev = result[0]; - for (int i = 1; i < size; i++) { - long zigzagEncoded = encodedData[i]; - long diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 - result[i] = prev + diff; - prev = result[i]; - } - - return result; - } - - public static void main(String[] args) throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); -// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP"; - File outputDir = new File(outputDirstr); - -// RLDecisionModel trainedModel = trainModel(20, csvFilePath); - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; -// if(!file.getName().equals("Stocks-DE.csv")) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 30; -// System.out.println(numbers.size()); - - - // 方法：强化学习 -// long modelStart = System.nanoTime(); - int modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - long[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - long[] scaledInts = sprintz(scalingInt); - - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - // 1. 找出当前8个元素中的最大值 - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / 8] = bitWidth; - } - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, 8,encodePos); - int cur_cost = encodePos[0]*8; -// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 - long duration = System.nanoTime() - startTime; - - - long startDecodeTime = System.nanoTime(); - long[] sprintz_encode_data = decompressWithOptimalPacking(res, paddedArray.length, 8); - long[] decode_data = sprintzDecode(sprintz_encode_data); - long endDecodeTime = System.nanoTime(); - long decodeDuration = endDecodeTime - startDecodeTime; - - modelDecodeTime += decodeDuration; - modelTime += (duration); - modelCost += cur_cost; -// if(i==0) -// for (int episode = 0; episode < 10; episode++) { -// trainEpisode(scaledInts, episode); -// } -// List optimalK = predictOptimalK(scaledInts); -// System.out.println("Optimal k sequence: " + optimalK); - } - - } - modelCost /=time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - modelDecodeTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); - double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); - String[] record = { - file.toString(), - "Sprintz-DP", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); -// break; - } - - } - - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); -// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP_vary_pack_size"; - File outputDir = new File(outputDirstr); - - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; -// if(!file.getName().equals("Stocks-DE.csv")) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - System.out.println(pack_size); - int modelCost = 0; - long modelTime = 0; - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - - long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - long[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking( paddedArray, bitWidths, pack_size,encodePos); - int cur_cost = encodePos[0]*8; - -// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; -// if(i==0) -// for (int episode = 0; episode < 10; episode++) { -// trainEpisode(scaledInts, episode); -// } -// List optimalK = predictOptimalK(scaledInts); -// System.out.println("Optimal k sequence: " + optimalK); - } - - } - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - String[] record = { - file.toString(), - "SPRINTZ-DP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - writer.close(); -// break; - } - - } - - // 新增方法：测试不同chunk size的表现 - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_DP_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 -// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); -// long[] scaledInts_all = scaleNumbers(numbers, decimalMax); - - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - long[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - long[] scaledInts_all = new long[totalLength]; - - int currentIndex = 0; - for (long[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); -// -// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - int end = Math.min(i + chunkSize, numbers.size()); - long[] scaledInt = new long[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - - long startTime = System.nanoTime(); - - long[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为pack_size的倍数 - long[] paddedArray = new long[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - long maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - int[] encodePos = new int[1]; - byte[] res = compressWithOptimalPacking(paddedArray, bitWidths, pack_size, encodePos); - int cur_cost = encodePos[0] * 8; - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "Sprintz-DP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); - } - } - -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java deleted file mode 100644 index 90eb6e422..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLP.java +++ /dev/null @@ -1,1321 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; -// EfficientOctadPackingMLP_optimized.java -// Java 17+ - -import org.junit.Test; - -import java.io.*; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.nio.file.*; -import java.util.*; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.*; - -public class EfficientOctadPackingMLP { - - - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - static final int CHUNK_SIZE = 1024; - static final int INPUT_DIM = 5; - static final int HIDDEN_DIM = 48; - - // ========== CSV loader & scaling helpers ========== - static List> loadDataFromCSV(String filename) { - List> sequences = new ArrayList<>(); - Pattern pattern = Pattern.compile("\"?\\[([0-9,\\s]+)\\]\"?"); - try (BufferedReader br = new BufferedReader(new FileReader(filename))) { - String line; - boolean firstLine = true; - while ((line = br.readLine()) != null) { - if (firstLine) { firstLine = false; continue; } - Matcher m = pattern.matcher(line); - if (m.find()) { - String data = m.group(1); - List arr = new ArrayList<>(); - String[] tokens = data.split(","); - for (String t : tokens) { - String s = t.trim(); - if (s.isEmpty()) continue; - try { - arr.add(Integer.parseInt(s)); - } catch (Exception ex) { /* ignore */ } - } - if (!arr.isEmpty()) sequences.add(arr); - } - } - } catch (IOException e) { - System.err.println("Error opening CSV: " + filename); - } - return sequences; - } - - static String trimStr(String s) { - if (s == null) return ""; - int a = 0; - while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; - if (a == s.length()) return ""; - int b = s.length() - 1; - while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; - return s.substring(a, b + 1); - } - - static String stripEnclosingQuotes(String s) { - if (s == null) return ""; - if (s.length() >= 2) { - char f = s.charAt(0); - char l = s.charAt(s.length() - 1); - if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { - return s.substring(1, s.length() - 1); - } - } - return s; - } - - // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping - static long[] scaleNumbers(List numbers, int decimalMax) { - int n = numbers.size(); - long[] result = new long[n]; - if (n == 0) return result; - - BigDecimal scale = BigDecimal.ONE; - for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); - - BigDecimal[] vals = new BigDecimal[n]; - for (int i = 0; i < n; ++i) { - String s = trimStr(numbers.get(i)); - s = stripEnclosingQuotes(s); - if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } - s = s.replace(",", ""); // remove thousands sep - - try { - BigDecimal bd = new BigDecimal(s); - BigDecimal scaled = bd.multiply(scale); - BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); - vals[i] = rounded; - } catch (Exception ex) { - try { - double dv = Double.parseDouble(s); - BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); - vals[i] = bd.setScale(0, RoundingMode.HALF_UP); - } catch (Exception ex2) { - System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); - vals[i] = BigDecimal.ZERO; - } - } - } - - BigDecimal minv = vals[0]; - for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; - - for (int i = 0; i < n; ++i) { - BigDecimal shifted = vals[i].subtract(minv); - try { - BigInteger bi = shifted.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (ArithmeticException ae) { - BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); - try { - BigInteger bi = rounded.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (Exception ex) { - result[i] = 0; - } - } - } - return result; - } - // ========== Pack / Result / DecisionPoint ========== - static class Pack { - int size = 0; - int maxBitWidth = 0; - int startIndex = 0; - List indices = new ArrayList<>(); - List bitWidths = new ArrayList<>(); - - void addOctad(int index, int bitWidth) { - if (size == 0) { - startIndex = index; - maxBitWidth = bitWidth; - } else { - if (bitWidth > maxBitWidth) maxBitWidth = bitWidth; - } - indices.add(index); - bitWidths.add(bitWidth); - size++; - } - - long dataCost(long pack_size) { - return pack_size * size * (long) maxBitWidth; - } - - int logSize() { - if (size <= 0) return 0; - return 32 - Integer.numberOfLeadingZeros(size); - } - } - - static class PackingResult { - int packCount = 0; - long dataCostA = 0; - int bitWidthCostB = 0; - int packSizeCostC = 0; - long totalCost = 0; - List packs = new ArrayList<>(); - byte[] compressedData; - - void calculateCost(int maxLog) { - bitWidthCostB = 6 * packCount; - packSizeCostC = packCount * maxLog; - totalCost = dataCostA + bitWidthCostB + packSizeCostC; - } - - @Override - public String toString() { - return String.format("Packs: %d, Cost: %d (A=%d, B=%d, C=%d)", packCount, totalCost, dataCostA, bitWidthCostB, packSizeCostC); - } - } - - private static double safeLog(double p) { - return Math.log(Math.max(p, 1e-8)); - } - - static class DecisionPoint { - int currentPackSize; - int currentPackMaxB; - int newOctadB; - int packCount; - int currentMaxLog; - boolean action; - float probability; - - DecisionPoint(int cps, int cpm, int nob, int pc, int cml, boolean a, float p) { - currentPackSize = cps; - currentPackMaxB = cpm; - newOctadB = nob; - packCount = pc; - currentMaxLog = cml; - action = a; - probability = p; - } - } - - // ========== 2-layer MLP policy with REINFORCE ========== - static class RLDecisionModel { - float[] W1; // size HIDDEN_DIM * INPUT_DIM - float[] b1; // size HIDDEN_DIM - float[] W2; // size HIDDEN_DIM - float b2; - - float explorationRate = 0.3f; - float learningRate = 0.01f; - - Random rng; - - RLDecisionModel() { - rng = new Random(); - W1 = new float[HIDDEN_DIM * INPUT_DIM]; - b1 = new float[HIDDEN_DIM]; - W2 = new float[HIDDEN_DIM]; - for (int i = 0; i < W1.length; ++i) W1[i] = randUniform(-0.08f, 0.08f); - for (int i = 0; i < b1.length; ++i) b1[i] = randUniform(-0.08f, 0.08f); - for (int i = 0; i < W2.length; ++i) W2[i] = randUniform(-0.08f, 0.08f); - b2 = randUniform(-0.08f, 0.08f); - } - - private float randUniform(float a, float b) { - return a + rng.nextFloat() * (b - a); - } - - static float relu(float x) { return x > 0.0f ? x : 0.0f; } - static float reluDeriv(float x) { return x > 0.0f ? 1.0f : 0.0f; } - static float sigmoid(float x) { - if (x >= 0) { - double z = Math.exp(-x); - return (float)(1.0 / (1.0 + z)); - } else { - double z = Math.exp(x); - return (float)(z / (1.0 + z)); - } - } - - float forwardProb(float[] feat, float[] outHidden, float[] outZ1) { - if (outHidden != null) Arrays.fill(outHidden, 0.0f); - if (outZ1 != null) Arrays.fill(outZ1, 0.0f); - - for (int h = 0; h < HIDDEN_DIM; ++h) { - float z = b1[h]; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) { - z += W1[base + j] * feat[j]; - } - if (outZ1 != null) outZ1[h] = z; - float hval = relu(z); - if (outHidden != null) outHidden[h] = hval; - } - - float z2 = b2; - if (outHidden != null) { - for (int h = 0; h < HIDDEN_DIM; ++h) z2 += W2[h] * outHidden[h]; - } else { - for (int h = 0; h < HIDDEN_DIM; ++h) { - float z = b1[h]; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) z += W1[base + j] * feat[j]; - float hval = relu(z); - z2 += W2[h] * hval; - } - } - return sigmoid(z2); - } - - float forwardProb(float[] feat) { - return forwardProb(feat, null, null); - } - - float train(List decisions, float reward) { - if (decisions == null || decisions.isEmpty()) return 0.0f; - - explorationRate *= 0.99f; - if (explorationRate < 0.05f) explorationRate = 0.05f; - - float[] dW1 = new float[W1.length]; - float[] db1 = new float[b1.length]; - float[] dW2 = new float[W2.length]; - float db2 = 0.0f; - - float totalLoss = 0.0f; - - float[] feat = new float[INPUT_DIM]; - float[] hidden = new float[HIDDEN_DIM]; - float[] z1 = new float[HIDDEN_DIM]; - - for (DecisionPoint dp : decisions) { - feat[0] = dp.currentPackSize / 100.0f; - feat[1] = dp.currentPackMaxB / 64.0f; - feat[2] = dp.newOctadB / 64.0f; - feat[3] = dp.packCount / 100.0f; - feat[4] = dp.currentMaxLog / 10.0f; - - float p = forwardProb(feat, hidden, z1); - - float pClipped = Math.min(Math.max(p, 1e-6f), 1.0f - 1e-6f); - - float piA = dp.action ? pClipped : (1.0f - pClipped); - if (piA <= 0.0f) { - piA = 1e-6f; - } - float lossI = -reward * (float) Math.log(piA); - - if (Float.isNaN(lossI) || Float.isInfinite(lossI)) { - System.err.printf("Warning: loss is NaN or Infinite. p=%.8f, piA=%.8f, reward=%.8f\n", p, piA, reward); - lossI = 0.0f; - } - - totalLoss += lossI; - - float dL_dz2 = reward * (p - (dp.action ? 1.0f : 0.0f)); - - for (int h = 0; h < HIDDEN_DIM; ++h) { - dW2[h] += dL_dz2 * hidden[h]; - } - db2 += dL_dz2; - - for (int h = 0; h < HIDDEN_DIM; ++h) { - float w2h = W2[h]; - float dh = dL_dz2 * w2h; - float dReLU = reluDeriv(z1[h]); - float dZ1 = dh * dReLU; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) { - dW1[base + j] += dZ1 * feat[j]; - } - db1[h] += dZ1; - } - } - - float lr = learningRate; - for (int i = 0; i < W1.length; ++i) { - W1[i] -= lr * dW1[i]; - W1[i] = clip(W1[i], -10f, 10f); - } - for (int i = 0; i < b1.length; ++i) { - b1[i] -= lr * db1[i]; - b1[i] = clip(b1[i], -10f, 10f); - } - for (int i = 0; i < W2.length; ++i) { - W2[i] -= lr * dW2[i]; - W2[i] = clip(W2[i], -10f, 10f); - } - b2 -= lr * db2; - b2 = clip(b2, -10f, 10f); - - return totalLoss; - } - - static float clip(float v, float low, float high) { - return Math.min(Math.max(v, low), high); - } - - } - - // ========== Bitpacking utility methods (legacy 8-values helpers kept) ========== - public static int getBitWidth(int num) { - if (num == 0) - return 1; - else - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - int leftBit = 0; - - while (valueIdx < 8 + offset) { - int buffer = 0; - int leftSize = 32; - - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - if (leftSize > 0 && valueIdx < 8 + offset) { - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - return encode_pos; - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - private static class BitWriter { - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - private long acc = 0L; // holds currently buffered bits (lowest "accBits" bits are valid) - private int accBits = 0; // number of bits in acc - - // writeBits expects bits in LSB-aligned form (i.e., "masked" value). We append MSB-first as: acc = (acc << bitCount) | bits - void writeBits(long bits, int bitCount) { - if (bitCount == 0) return; - if (bitCount == 64) { - // split into two 32-bit writes to avoid shifting by 64 - writeBits((bits >>> 32) & 0xFFFFFFFFL, 32); - writeBits(bits & 0xFFFFFFFFL, 32); - return; - } - long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); - long v = bits & mask; - acc = (acc << bitCount) | v; - accBits += bitCount; - while (accBits >= 8) { - int shift = accBits - 8; - int outb = (int) ((acc >>> shift) & 0xFFL); - out.write(outb); - if (shift > 0) { - acc &= ((1L << shift) - 1L); - } else { - acc = 0L; - } - accBits = shift; - } - } - - byte[] finish() { - if (accBits > 0) { - int outb = (int) ((acc << (8 - accBits)) & 0xFFL); - out.write(outb); - acc = 0L; - accBits = 0; - } - return out.toByteArray(); - } - } -// private static class BitReader { -// final byte[] data; -// private long acc = 0L; -// private int accBits = 0; -// private int idx = 0; -// -// BitReader(byte[] data) { -// this.data = data; -// } -// -// long readBits(int bitCount) throws IOException { -// if (bitCount == 0) return 0L; -// while (accBits < bitCount) { -// if (idx < data.length) { -// acc = (acc << 8) | (data[idx++] & 0xFFL); -// accBits += 8; -// } else { -// // pad with zeros if stream ends prematurely -// acc = (acc << (bitCount - accBits)); -// accBits = bitCount; -// } -// } -// int shift = accBits - bitCount; -// long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); -// long v = (acc >>> shift) & mask; -// if (shift > 0) { -// acc &= ((1L << shift) - 1L); -// } else { -// acc = 0L; -// } -// accBits = shift; -// return v; -// } -// } -public static final class BitReader { - private final byte[] data; - private int bitPos; // global bit position from start of data[] - - public BitReader(byte[] data) { - this(data, 0); - } - - public BitReader(byte[] data, int byteOffset) { - this.data = data; - this.bitPos = byteOffset * 8; - } - - /** - * Read n bits (0 <= n <= 64), return as unsigned long. - */ - public long readBits(int n) { - if (n == 0) return 0L; - if (n < 0 || n > 64) { - throw new IllegalArgumentException("n must be between 0 and 64"); - } - - long result = 0L; - int bitsRemaining = n; - - while (bitsRemaining > 0) { - int byteIndex = bitPos >>> 3; // current byte - int bitOffset = bitPos & 7; // offset inside byte [0..7] - - // 添加边界检查 - if (byteIndex >= data.length) { - // 如果已经超出数据范围，填充0并返回 - result = (result << bitsRemaining); - bitPos += bitsRemaining; - return result; - } - - int bitsFromCurrentByte = Math.min(8 - bitOffset, bitsRemaining); - - // Load byte as unsigned - int curByte = data[byteIndex] & 0xFF; - - // Shift to get the relevant bits - int shift = 8 - bitOffset - bitsFromCurrentByte; - int chunk = (curByte >>> shift) & ((1 << bitsFromCurrentByte) - 1); - - result = (result << bitsFromCurrentByte) | chunk; - - bitPos += bitsFromCurrentByte; - bitsRemaining -= bitsFromCurrentByte; - } - - return result; - } - - /** - * @return total bits consumed since creation / since byteOffset - */ - public int consumedBits() { - return bitPos; - } - - /** - * @return current bit position (alias) - */ - public int bitPosition() { - return bitPos; - } - - /** - * @return remaining bits available for reading - */ - public int remainingBits() { - return (data.length * 8) - bitPos; - } -} - private static byte[] performBitPackingCompression64_fast(long[] dataArray, List packs, int pack_size, int originalLength) throws IOException { - // 计算一些元信息 - int totalPacks = packs.size(); - int maxOctadsInAnyPack = 0; - for (Pack p : packs) if (p.size > maxOctadsInAnyPack) maxOctadsInAnyPack = p.size; - - // bits needed to encode counts in range [0..maxOctadsInAnyPack] - int bitsForCount = 1; - while ((1L << bitsForCount) <= maxOctadsInAnyPack) bitsForCount++; - if (bitsForCount <= 0) bitsForCount = 1; - - // 准备输出缓冲（先写 header 的整数字段，meta/data 用 BitWriter 位流） - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - - dos.writeByte(totalPacks); - dos.writeByte(bitsForCount); // 1 byte is enough to carry this small number - dos.flush(); - - // --- Meta bitstream --- - BitWriter metaWriter = new BitWriter(); - - // For each pack: write pack.size using bitsForCount bits, then for each octad write its bitWidth using 6 bits - for (Pack pack : packs) { - // write octad count - metaWriter.writeBits(pack.size, bitsForCount); - metaWriter.writeBits(pack.bitWidths.get(0), 6); -// // write each octad's bitWidth using 6 bits. -// // NOTE: we map bitWidth==64 -> store 63 (as sentinel). 解码端须按此约定把 63 映射回 64。 -// for (int i = 0; i < pack.size; ++i) { -// int bw = pack.bitWidths.get(i); -// int store = bw; -// if (bw == 64) store = 63; -// if (store < 0) store = 0; -// if (store > 63) store = 63; // safety clamp -// metaWriter.writeBits(store, 5); -// } - } - - byte[] metaBytes = metaWriter.finish(); - dos.write(metaBytes); - dos.flush(); - - // --- Data bitstream --- - BitWriter dataWriter = new BitWriter(); - - // For each pack, find packMaxBitWidth and write each group's pack_size values using packMaxBitWidth bits - for (Pack pack : packs) { - int packMaxBW = pack.maxBitWidth; - // no change for packMaxBW == 64: BitWriter supports splitting 64 into two 32-bit writes - for (int i = 0; i < pack.size; ++i) { - int originalGroupIndex = pack.indices.get(0); - int startPos = originalGroupIndex * pack_size; - for (int j = 0; j < pack_size; ++j) { - long val; -// if (startPos + j < dataArray.length) { - val = dataArray[startPos + j]; -// } else { -// val = 0L; -// } - // mask value to packMaxBW bits (if packMaxBW == 64, mask preserves full 64 bits) - long mask; - if (packMaxBW == 0) { - dataWriter.writeBits(0L, 0); // nothing to write - } else { - if (packMaxBW == 64) { - // write full 64-bit value (BitWriter handles split) - dataWriter.writeBits(val, 64); - } else { - mask = (1L << packMaxBW) - 1L; - long masked = val & mask; - dataWriter.writeBits(masked, packMaxBW); - } - } - } - } - } - - byte[] dataBytes = dataWriter.finish(); - dos.write(dataBytes); - dos.flush(); - - return baos.toByteArray(); - } - - // ========== packOctads (updated to accept originalLength for compression) ========== - static PackingResult packOctads(List bitWidths, RLDecisionModel model, List decisionTrace, int pack_size, long[] dataArray, int originalLength) { - PackingResult result = new PackingResult(); - Pack currentPack = new Pack(); - int globalMaxLog = 0; - int packCount = 0; - - Random localRng = ThreadLocalRandom.current(); - - for (int i = 0; i < bitWidths.size(); ++i) { - int b = bitWidths.get(i); - - if (currentPack.size == 0) { - currentPack.addOctad(i, b); - } else if (b == currentPack.maxBitWidth) { - currentPack.addOctad(i, b); - } else { - float[] feat = new float[INPUT_DIM]; - feat[0] = currentPack.size / 100.0f; - feat[1] = currentPack.maxBitWidth / 64.0f; - feat[2] = b / 64.0f; - feat[3] = packCount / 100.0f; - feat[4] = globalMaxLog / 10.0f; - - float probability = model.forwardProb(feat); - - boolean shouldMerge; - if (localRng.nextFloat() < model.explorationRate) { - shouldMerge = (localRng.nextFloat() > 0.5f); - } else { - shouldMerge = probability > 0.5f; - } - - if (decisionTrace != null) { - decisionTrace.add(new DecisionPoint(currentPack.size, currentPack.maxBitWidth, b, packCount, globalMaxLog, shouldMerge, probability)); - } - - if (shouldMerge) { - currentPack.addOctad(i, b); - } else { - result.dataCostA += currentPack.dataCost(pack_size); - int logSize = currentPack.logSize(); - if (logSize > globalMaxLog) globalMaxLog = logSize; - result.packs.add(currentPack); - packCount++; - - currentPack = new Pack(); - currentPack.addOctad(i, b); - } - } - } - - if (currentPack.size > 0) { - result.dataCostA += currentPack.dataCost(pack_size); - int logSize = currentPack.logSize(); - if (logSize > globalMaxLog) globalMaxLog = logSize; - result.packs.add(currentPack); - packCount++; - } - - result.packCount = packCount; - result.calculateCost(globalMaxLog); - - // 执行实际的bitpacking压缩（如果提供了 dataArray） - if (dataArray != null) { - try { - result.compressedData = performBitPackingCompression64_fast(dataArray, result.packs, pack_size, originalLength); - } catch (IOException e) { - System.err.println("Compression failed: " + e.getMessage()); - result.compressedData = null; - } - } - - return result; - } - -// public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { -// try { -// // 这里需要根据实际的压缩格式来解析 -// // 假设compressedData包含bit-packed数据 -// ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); -// DataInputStream dis = new DataInputStream(bais); -// -// // 读取bit-packed数据 -// int totalGroups = bitWidths.length; -// long[] result = new long[totalGroups * packSize]; -// int resultIndex = 0; -// -// BitReader reader = new BitReader(compressedData); -// -// for (int g = 0; g < totalGroups; ++g) { -// int bw = bitWidths[g]; -// for (int k = 0; k < packSize; ++k) { -// if (bw == 0) { -// result[resultIndex++] = 0L; -// } else if (bw == 64) { -// long high = reader.readBits(32); -// long low = reader.readBits(32); -// long v = (high << 32) | (low & 0xFFFFFFFFL); -// result[resultIndex++] = v; -// } else { -// long v = reader.readBits(bw); -// result[resultIndex++] = v; -// } -// } -// } -// -// // 只取原始长度的数据并Sprintz解码 -// return Arrays.copyOf(result, originalLength); -// -// } catch (IOException e) { -// System.err.println("Fast decompression failed: " + e.getMessage()); -// return new long[0]; -// } -// } - - public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { - if (compressedData == null || compressedData.length == 0) { - System.err.println("Compressed data is null or empty"); - return new long[0]; - } - - try { - ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); - DataInputStream dis = new DataInputStream(bais); - - // === Header === - int totalPacks = dis.readUnsignedByte(); - int bitsForCount = dis.readUnsignedByte(); - - // === Read meta bitstream === - int metaStartOffset = 2; // two bytes read - BitReader metaReader = new BitReader(compressedData, metaStartOffset); - - // 解析每个pack的信息 - List packInfos = new ArrayList<>(); - int totalValuesToDecode = 0; - - for (int p = 0; p < totalPacks; ++p) { - int octadCount = (int) metaReader.readBits(bitsForCount); - int packBitWidth = (int) metaReader.readBits(6); - if (packBitWidth == 63) packBitWidth = 64; - - packInfos.add(new PackInfo(octadCount, packBitWidth)); - totalValuesToDecode += octadCount * packSize; - } - - // 计算数据部分的起始位置 - int metaBitsUsed = metaReader.consumedBits(); - int dataStartByte = metaStartOffset + (metaBitsUsed + 7) / 8; - - // 检查数据起始位置是否超出压缩数据范围 - if (dataStartByte >= compressedData.length) { -// System.err.println("Data start position exceeds compressed data length"); - return new long[0]; - } - - // === Data bitstream === - BitReader dataReader = new BitReader(compressedData, dataStartByte); - List resultList = new ArrayList<>(); - - // === 按pack解码数据 === - for (PackInfo packInfo : packInfos) { - int octadCount = packInfo.octadCount; - int bitWidth = packInfo.bitWidth; - - // 检查剩余数据是否足够 - if (dataReader.remainingBits() < (long) octadCount * packSize * bitWidth) { -// System.err.println("Insufficient data for decoding pack. Expected: " + -// (octadCount * packSize * bitWidth) + " bits, Available: " + -// dataReader.remainingBits() + " bits"); - break; - } - - // 每个octad包含packSize个值 - for (int i = 0; i < octadCount; ++i) { - for (int j = 0; j < packSize; ++j) { - long value; - if (bitWidth == 0) { - value = 0L; - } else if (bitWidth == 64) { - // 64位特殊处理：分成两个32位读取 - long high = dataReader.readBits(32); - long low = dataReader.readBits(32); - value = (high << 32) | low; - } else { - value = dataReader.readBits(bitWidth); - } - resultList.add(value); - } - } - } - - // 转换为数组并截取到原始长度 - long[] result = new long[Math.min(resultList.size(), originalLength)]; - for (int i = 0; i < result.length; i++) { - result[i] = resultList.get(i); - } - -// System.out.println("Decompression completed: " + result.length + " values decoded"); - return result; - - } catch (Exception e) { - System.err.println("Fast decompression failed: " + e.getMessage()); - e.printStackTrace(); - return new long[0]; - } - } - - // 辅助类，存储pack信息 - static class PackInfo { - int octadCount; - int bitWidth; - - PackInfo(int octadCount, int bitWidth) { - this.octadCount = octadCount; - this.bitWidth = bitWidth; - } - } - // ========== Training loop (trainModel) ========== - static RLDecisionModel trainModel(int epochs, String csvFilePath) { - System.err.println("Training RL model from CSV data..."); - RLDecisionModel model = new RLDecisionModel(); - List> sequences = loadDataFromCSV(csvFilePath); - if (sequences.isEmpty()) { - System.err.println("No data loaded from CSV. Returning initial model."); - return model; - } - System.err.println("Loaded " + sequences.size() + " sequences from CSV"); - - List decisionTrace = new ArrayList<>(); - for (int epoch = 1; epoch <= epochs; ++epoch) { - long startTime = System.nanoTime(); - float totalReward = 0.0f; - float totalLoss = 0.0f; - int processedSequences = 0; - - for (List bitWidths : sequences) { - decisionTrace.clear(); - // training does not perform actual compression, pass dataArray=null and originalLength=0 - PackingResult result = packOctads(bitWidths, model, decisionTrace, 8, null, 0); - - float reward = (float) result.totalCost / 500000.0f; - totalReward += reward; - - float loss = model.train(decisionTrace, reward); - totalLoss += loss; - processedSequences++; - } - - long durationMs = (System.nanoTime() - startTime) / 1_000_000L; - - if (epoch % 10 == 0 || epoch == 1 || epoch == epochs) { - System.out.printf("Epoch %d: Avg Reward = %.6f, Avg Loss = %.6f, Time = %d ms%n", - epoch, - totalReward / processedSequences, - totalLoss / processedSequences, - durationMs); - } else { - System.out.printf("Epoch %d done. Time = %d ms%n", epoch, durationMs); - } - } - - return model; - } - - - // ========== performanceTest (更新版本，包含解压测试) ========== - static void performanceTest(RLDecisionModel model, String directory, String outputDirStr) { - System.out.println("\nPerformance Testing..."); - Path outdir = Paths.get(outputDirStr); - try { - if (!Files.exists(outdir)) Files.createDirectories(outdir); - } catch (IOException e) { - System.err.println("Cannot create output dir: " + outputDirStr); - return; - } - - try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { - for (Path entry : ds) { - if (!Files.isRegularFile(entry)) continue; - String fname = entry.getFileName().toString(); - if (IGNORE_FILES.contains(fname)) continue; - - System.out.println("Processing " + fname + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - - try (BufferedReader br = Files.newBufferedReader(entry)) { - String line; - while ((line = br.readLine()) != null) { - String[] tokens = line.split(","); - for (String token : tokens) { - String t = trimStr(token); - if (!t.isEmpty()) { - numbers.add(t); - int dec = 0; - int pos = t.indexOf('.'); - if (pos != -1) dec = t.length() - pos - 1; - decimalPlaces.add(dec); - } - } - } - } catch (IOException e) { - System.err.println("Cannot open " + entry.toString()); - continue; - } - - if (numbers.isEmpty()) continue; - - Path outPath = outdir.resolve(fname); - try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { - // 更新表头，增加解压吞吐率列 - writer.write("Input Direction,Encoding Algorithm,Encoding Time,Decoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); - - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; // 新增：解压时间统计 - long compressedSize = 0; - - for (int rep = 0; rep < time_of_repeat; ++rep) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(numbers.size(), i + CHUNK_SIZE); - if (end - i <= 2) continue; - List chunkNumbers = numbers.subList(i, end); - int decimalMax = 0; - for (int k = i; k < end; ++k) { - if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); - } - - long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - - int remainder = scaledInts.length % pack_size; - int padding = (remainder == 0) ? 0 : pack_size - remainder; - long[] padded = new long[scaledInts.length + padding]; - System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); - if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); - - int groups = padded.length / pack_size; - int[] bitWidths = new int[groups]; - int gidx = 0; - for (int si = 0; si < padded.length; si += pack_size) { - long maxInGroup = 0; - for (int sj = si; sj < si + pack_size; ++sj) { - long v = padded[sj]; - if (v > maxInGroup) maxInGroup = v; - } - int bitWidth = 0; - if (maxInGroup > 0) { - bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - } else { - bitWidth = 0; - } - bitWidths[gidx++] = bitWidth; - } - - // pass original length (un-padded) so decoder can trim - List bitWidthsList = new ArrayList<>(groups); - for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); - - PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += (res.compressedData.length*8); - - // 新增：测试解压性能 - if (res.compressedData != null) { - long startDecodeTime = System.nanoTime(); - long[] decompressed = fastDecompress(res.compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - } - -// if (rep == 0) { -// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; -// } - } - } - - modelCost /= time_of_repeat; - modelTime /= time_of_repeat; - modelDecodeTime /= time_of_repeat; // 平均解压时间 - - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes - double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) modelTime; // points/s - double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) modelDecodeTime; // points/s - - writer.write(entry + ","); - writer.write("BP-RL,"); - writer.write(modelTime_throughput + ","); - writer.write(modelDecodeTime_throughput + ","); // 解压吞吐率 - writer.write(numbers.size() + ","); - writer.write(modelCost + ","); - writer.write(pack_size + ","); - writer.write(model_ratio + "\n"); - -// System.out.println("Pack Size: " + pack_size); -// System.out.println("Encoding throughput: " + modelTime_throughput + " points/s"); -// System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " points/s"); -// System.out.println("Compression ratio: " + model_ratio); - } - } catch (IOException e) { - System.err.println("Error writing output file for " + fname); - } - } - } catch (IOException e) { - System.err.println("Error iterating directory: " + directory); - } - } - - public static void main(String[] args) { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL"; - - int epochs = 20; - - if (args.length >= 1) trainCsv = args[0]; - if (args.length >= 2) dataDir = args[1]; - if (args.length >= 3) outDir = args[2]; - - RLDecisionModel model = new RLDecisionModel(); - if (!trainCsv.isEmpty()) { - model = trainModel(epochs, trainCsv); - } else { - System.err.println("No training CSV given. Using randomly initialized RL model."); - } - - if (!dataDir.isEmpty()) { - performanceTest(model, dataDir, outDir); - } else { - System.err.println("No data directory provided for performanceTest. Exiting."); - } - } - @Test - public void TestVarPackSize() { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL_vary_pack_size"; - - int epochs = 80; - - RLDecisionModel model = new RLDecisionModel(); - model = trainModel(epochs, trainCsv); - performanceTest(model, dataDir, outDir); - } - // ========== performanceTest with variable chunk sizes ========== - static void performanceTestVariableChunkSize(RLDecisionModel model, String directory, String outputDirStr) { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - Path outdir = Paths.get(outputDirStr); - try { - if (!Files.exists(outdir)) Files.createDirectories(outdir); - } catch (IOException e) { - System.err.println("Cannot create output dir: " + outputDirStr); - return; - } - - // Define the chunk sizes to test (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { - for (Path entry : ds) { - if (!Files.isRegularFile(entry)) continue; - String fname = entry.getFileName().toString(); - if (IGNORE_FILES.contains(fname)) continue; - - System.out.println("Processing " + fname + " with variable chunk sizes..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - - try (BufferedReader br = Files.newBufferedReader(entry)) { - String line; - while ((line = br.readLine()) != null) { - String[] tokens = line.split(","); - for (String token : tokens) { - String t = trimStr(token); - if (!t.isEmpty()) { - numbers.add(t); - int dec = 0; - int pos = t.indexOf('.'); - if (pos != -1) dec = t.length() - pos - 1; - decimalPlaces.add(dec); - } - } - } - } catch (IOException e) { - System.err.println("Cannot open " + entry.toString()); - continue; - } - - if (numbers.isEmpty()) continue; - - Path outPath = outdir.resolve(fname.replace(".", "_chunksize_test.")); - try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { - writer.write("m,Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); - - int time_of_repeat = 50; // Reduced for faster testing with multiple chunk sizes - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - long[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - long[] scaledInts_all = new long[totalLength]; - - int currentIndex = 0; - for (long[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - - // Test each chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for (int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - long modelCost = 0; - long modelTime = 0; - long compressedSize = 0; - - for (int rep = 0; rep < time_of_repeat; ++rep) { - for (int i = 0; i < numbers.size(); i += chunkSize) { -// int end = Math.min(numbers.size(), i + chunkSize); -// if (end - i <= 2) continue; -// List chunkNumbers = numbers.subList(i, end); -// int decimalMax = 0; -// for (int k = i; k < end; ++k) { -// if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); -// } -// -// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - int end = Math.min(i + chunkSize, numbers.size()); - long[] scaledInts = new long[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - - int remainder = scaledInts.length % pack_size; - int padding = (remainder == 0) ? 0 : pack_size - remainder; - long[] padded = new long[scaledInts.length + padding]; - System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); - if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); - - int groups = padded.length / pack_size; - int[] bitWidths = new int[groups]; - int gidx = 0; - for (int si = 0; si < padded.length; si += pack_size) { - long maxInGroup = 0; - for (int sj = si; sj < si + pack_size; ++sj) { - long v = padded[sj]; - if (v > maxInGroup) maxInGroup = v; - } - int bitWidth = 0; - if (maxInGroup > 0) { - bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - } else { - bitWidth = 0; - } - bitWidths[gidx++] = bitWidth; - } - - // pass original length (un-padded) so decoder can trim - List bitWidthsList = new ArrayList<>(groups); - for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); - - PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += (res.compressedData.length* 8L); - -// if (rep == 0) { -// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; -// } - } - } - - modelCost /= time_of_repeat; - modelTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms - - writer.write(String.valueOf(chunkSize/8) + ","); - writer.write(entry.toString() + ","); - writer.write("BP-RL,"); - writer.write(String.valueOf(modelTime_throughput) + ","); - writer.write(String.valueOf(numbers.size()) + ","); - writer.write(String.valueOf(modelCost) + ","); - writer.write(String.valueOf(pack_size) + ","); - writer.write(String.valueOf(model_ratio) + "\n"); - } - } - } catch (IOException e) { - System.err.println("Error writing output file for " + fname); - } - } - } catch (IOException e) { - System.err.println("Error iterating directory: " + directory); - } - } - - @Test - public void TestVariableChunkSize() { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRL_vary_m"; - - int epochs = 20; - - RLDecisionModel model = new RLDecisionModel(); - model = trainModel(epochs, trainCsv); - performanceTestVariableChunkSize(model, dataDir, outDir); - } -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java deleted file mode 100644 index b0a0f8c4b..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/EfficientOctadPackingMLPSprintz.java +++ /dev/null @@ -1,1504 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; -// EfficientOctadPackingMLP_optimized.java -// Java 17+ - -import org.junit.Test; - -import java.io.*; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.nio.file.*; -import java.util.*; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.*; - -public class EfficientOctadPackingMLPSprintz { - - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - static final int CHUNK_SIZE = 1024; - static final int INPUT_DIM = 5; - static final int HIDDEN_DIM = 48; - - // ========== Pack / Result / DecisionPoint ========== - static class Pack { - int size = 0; - int maxBitWidth = 0; - int startIndex = 0; - List indices = new ArrayList<>(); - List bitWidths = new ArrayList<>(); - - void addOctad(int index, int bitWidth) { - if (size == 0) { - startIndex = index; - maxBitWidth = bitWidth; - } else { - if (bitWidth > maxBitWidth) maxBitWidth = bitWidth; - } - indices.add(index); - bitWidths.add(bitWidth); - size++; - } - - long dataCost(long pack_size) { - return pack_size * size * (long) maxBitWidth; - } - - int logSize() { - if (size <= 0) return 0; - return 32 - Integer.numberOfLeadingZeros(size); - } - } - - static class PackingResult { - int packCount = 0; - long dataCostA = 0; - int bitWidthCostB = 0; - int packSizeCostC = 0; - long totalCost = 0; - List packs = new ArrayList<>(); - byte[] compressedData; - - void calculateCost(int maxLog) { - bitWidthCostB = 6 * packCount; - packSizeCostC = packCount * maxLog; - totalCost = dataCostA + bitWidthCostB + packSizeCostC; - } - - @Override - public String toString() { - return String.format("Packs: %d, Cost: %d (A=%d, B=%d, C=%d)", packCount, totalCost, dataCostA, bitWidthCostB, packSizeCostC); - } - } - - private static double safeLog(double p) { - return Math.log(Math.max(p, 1e-8)); - } - - static class DecisionPoint { - int currentPackSize; - int currentPackMaxB; - int newOctadB; - int packCount; - int currentMaxLog; - boolean action; - float probability; - - DecisionPoint(int cps, int cpm, int nob, int pc, int cml, boolean a, float p) { - currentPackSize = cps; - currentPackMaxB = cpm; - newOctadB = nob; - packCount = pc; - currentMaxLog = cml; - action = a; - probability = p; - } - } - - // ========== 2-layer MLP policy with REINFORCE ========== - static class RLDecisionModel { - float[] W1; // size HIDDEN_DIM * INPUT_DIM - float[] b1; // size HIDDEN_DIM - float[] W2; // size HIDDEN_DIM - float b2; - - float explorationRate = 0.3f; - float learningRate = 0.01f; - - Random rng; - - RLDecisionModel() { - rng = new Random(); - W1 = new float[HIDDEN_DIM * INPUT_DIM]; - b1 = new float[HIDDEN_DIM]; - W2 = new float[HIDDEN_DIM]; - for (int i = 0; i < W1.length; ++i) W1[i] = randUniform(-0.08f, 0.08f); - for (int i = 0; i < b1.length; ++i) b1[i] = randUniform(-0.08f, 0.08f); - for (int i = 0; i < W2.length; ++i) W2[i] = randUniform(-0.08f, 0.08f); - b2 = randUniform(-0.08f, 0.08f); - } - - private float randUniform(float a, float b) { - return a + rng.nextFloat() * (b - a); - } - - static float relu(float x) { return x > 0.0f ? x : 0.0f; } - static float reluDeriv(float x) { return x > 0.0f ? 1.0f : 0.0f; } - static float sigmoid(float x) { - if (x >= 0) { - double z = Math.exp(-x); - return (float)(1.0 / (1.0 + z)); - } else { - double z = Math.exp(x); - return (float)(z / (1.0 + z)); - } - } - - float forwardProb(float[] feat, float[] outHidden, float[] outZ1) { - if (outHidden != null) Arrays.fill(outHidden, 0.0f); - if (outZ1 != null) Arrays.fill(outZ1, 0.0f); - - for (int h = 0; h < HIDDEN_DIM; ++h) { - float z = b1[h]; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) { - z += W1[base + j] * feat[j]; - } - if (outZ1 != null) outZ1[h] = z; - float hval = relu(z); - if (outHidden != null) outHidden[h] = hval; - } - - float z2 = b2; - if (outHidden != null) { - for (int h = 0; h < HIDDEN_DIM; ++h) z2 += W2[h] * outHidden[h]; - } else { - for (int h = 0; h < HIDDEN_DIM; ++h) { - float z = b1[h]; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) z += W1[base + j] * feat[j]; - float hval = relu(z); - z2 += W2[h] * hval; - } - } - return sigmoid(z2); - } - - float forwardProb(float[] feat) { - return forwardProb(feat, null, null); - } - - float train(List decisions, float reward) { - if (decisions == null || decisions.isEmpty()) return 0.0f; - - explorationRate *= 0.99f; - if (explorationRate < 0.05f) explorationRate = 0.05f; - - float[] dW1 = new float[W1.length]; - float[] db1 = new float[b1.length]; - float[] dW2 = new float[W2.length]; - float db2 = 0.0f; - - float totalLoss = 0.0f; - - float[] feat = new float[INPUT_DIM]; - float[] hidden = new float[HIDDEN_DIM]; - float[] z1 = new float[HIDDEN_DIM]; - - for (DecisionPoint dp : decisions) { - feat[0] = dp.currentPackSize / 100.0f; - feat[1] = dp.currentPackMaxB / 64.0f; - feat[2] = dp.newOctadB / 64.0f; - feat[3] = dp.packCount / 100.0f; - feat[4] = dp.currentMaxLog / 10.0f; - - float p = forwardProb(feat, hidden, z1); - - float pClipped = Math.min(Math.max(p, 1e-6f), 1.0f - 1e-6f); - - float piA = dp.action ? pClipped : (1.0f - pClipped); - if (piA <= 0.0f) { - piA = 1e-6f; - } - float lossI = -reward * (float) Math.log(piA); - - if (Float.isNaN(lossI) || Float.isInfinite(lossI)) { - System.err.printf("Warning: loss is NaN or Infinite. p=%.8f, piA=%.8f, reward=%.8f\n", p, piA, reward); - lossI = 0.0f; - } - - totalLoss += lossI; - - float dL_dz2 = reward * (p - (dp.action ? 1.0f : 0.0f)); - - for (int h = 0; h < HIDDEN_DIM; ++h) { - dW2[h] += dL_dz2 * hidden[h]; - } - db2 += dL_dz2; - - for (int h = 0; h < HIDDEN_DIM; ++h) { - float w2h = W2[h]; - float dh = dL_dz2 * w2h; - float dReLU = reluDeriv(z1[h]); - float dZ1 = dh * dReLU; - int base = h * INPUT_DIM; - for (int j = 0; j < INPUT_DIM; ++j) { - dW1[base + j] += dZ1 * feat[j]; - } - db1[h] += dZ1; - } - } - - float lr = learningRate; - for (int i = 0; i < W1.length; ++i) { - W1[i] -= lr * dW1[i]; - W1[i] = clip(W1[i], -10f, 10f); - } - for (int i = 0; i < b1.length; ++i) { - b1[i] -= lr * db1[i]; - b1[i] = clip(b1[i], -10f, 10f); - } - for (int i = 0; i < W2.length; ++i) { - W2[i] -= lr * dW2[i]; - W2[i] = clip(W2[i], -10f, 10f); - } - b2 -= lr * db2; - b2 = clip(b2, -10f, 10f); - - return totalLoss; - } - - static float clip(float v, float low, float high) { - return Math.min(Math.max(v, low), high); - } - - } - - // ========== Bitpacking utility methods (legacy 8-values helpers kept) ========== - public static int getBitWidth(int num) { - if (num == 0) - return 1; - else - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - int leftBit = 0; - - while (valueIdx < 8 + offset) { - int buffer = 0; - int leftSize = 32; - - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - if (leftSize > 0 && valueIdx < 8 + offset) { - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - return encode_pos; - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - // ========== 64-bit-capable canonical encoder/decoder (fast, primitive-based) ========== - // DecodedResult wraps decoded padded values and originalLength (so caller can trim). - public static class DecodedResult { - public final long[] values; // padded values: totalGroups * packSize - public final int originalLength; // original (unpadded) length - public final int packSize; - public DecodedResult(long[] values, int originalLength, int packSize) { - this.values = values; - this.originalLength = originalLength; - this.packSize = packSize; - } - } - - - - - // ========== CSV loader & scaling helpers ========== - static List> loadDataFromCSV(String filename) { - List> sequences = new ArrayList<>(); - Pattern pattern = Pattern.compile("\"?\\[([0-9,\\s]+)\\]\"?"); - try (BufferedReader br = new BufferedReader(new FileReader(filename))) { - String line; - boolean firstLine = true; - while ((line = br.readLine()) != null) { - if (firstLine) { firstLine = false; continue; } - Matcher m = pattern.matcher(line); - if (m.find()) { - String data = m.group(1); - List arr = new ArrayList<>(); - String[] tokens = data.split(","); - for (String t : tokens) { - String s = t.trim(); - if (s.isEmpty()) continue; - try { - arr.add(Integer.parseInt(s)); - } catch (Exception ex) { /* ignore */ } - } - if (!arr.isEmpty()) sequences.add(arr); - } - } - } catch (IOException e) { - System.err.println("Error opening CSV: " + filename); - } - return sequences; - } - - static String trimStr(String s) { - if (s == null) return ""; - int a = 0; - while (a < s.length() && Character.isWhitespace(s.charAt(a))) a++; - if (a == s.length()) return ""; - int b = s.length() - 1; - while (b >= 0 && Character.isWhitespace(s.charAt(b))) b--; - return s.substring(a, b + 1); - } - - static String stripEnclosingQuotes(String s) { - if (s == null) return ""; - if (s.length() >= 2) { - char f = s.charAt(0); - char l = s.charAt(s.length() - 1); - if ((f == '"' && l == '"') || (f == '\'' && l == '\'')) { - return s.substring(1, s.length() - 1); - } - } - return s; - } - - // scaleNumbers: use BigDecimal to parse, scale by 10^decimalMax, shift so min becomes 0, return long[] with clipping - static long[] scaleNumbers(List numbers, int decimalMax) { - int n = numbers.size(); - long[] result = new long[n]; - if (n == 0) return result; - - BigDecimal scale = BigDecimal.ONE; - for (int i = 0; i < decimalMax; ++i) scale = scale.multiply(BigDecimal.TEN); - - BigDecimal[] vals = new BigDecimal[n]; - for (int i = 0; i < n; ++i) { - String s = trimStr(numbers.get(i)); - s = stripEnclosingQuotes(s); - if (s.isEmpty()) { vals[i] = BigDecimal.ZERO; continue; } - s = s.replace(",", ""); // remove thousands sep - - try { - BigDecimal bd = new BigDecimal(s); - BigDecimal scaled = bd.multiply(scale); - BigDecimal rounded = scaled.setScale(0, RoundingMode.HALF_UP); - vals[i] = rounded; - } catch (Exception ex) { - try { - double dv = Double.parseDouble(s); - BigDecimal bd = BigDecimal.valueOf(dv).multiply(scale); - vals[i] = bd.setScale(0, RoundingMode.HALF_UP); - } catch (Exception ex2) { - System.err.println("Warning: cannot parse token '" + numbers.get(i) + "', set to 0"); - vals[i] = BigDecimal.ZERO; - } - } - } - - BigDecimal minv = vals[0]; - for (int i = 1; i < n; ++i) if (vals[i].compareTo(minv) < 0) minv = vals[i]; - - for (int i = 0; i < n; ++i) { - BigDecimal shifted = vals[i].subtract(minv); - try { - BigInteger bi = shifted.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (ArithmeticException ae) { - BigDecimal rounded = shifted.setScale(0, RoundingMode.HALF_UP); - try { - BigInteger bi = rounded.toBigIntegerExact(); - if (bi.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) result[i] = Long.MAX_VALUE; - else if (bi.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) result[i] = Long.MIN_VALUE; - else result[i] = bi.longValue(); - } catch (Exception ex) { - result[i] = 0; - } - } - } - return result; - } - public static long[] sprintz(long[] numbers) { - int size = numbers.length; - long[] result = new long[size]; - - if (size == 0) return result; // 空数组直接返回 - - long first = numbers[0]; - result[0] = first; - - long prev = first; - for (int i = 1; i < size; i++) { - long current = numbers[i]; - long diff = current - prev; - // ZigZag 编码: 正数 -> 偶数, 负数 -> 奇数 - result[i] = (diff << 1) ^ (diff >> 63); - prev = current; - } - - return result; - } - /** - * Sprintz解码 - 从差分编码恢复原始数据 (long版本) - */ - public static long[] sprintzDecode(long[] encodedData) { - int size = encodedData.length; - long[] result = new long[size]; - - if (size == 0) return result; - - // 第一个元素是原始值 - result[0] = encodedData[0]; - - // 后续元素需要ZigZag解码和累加 - long prev = result[0]; - for (int i = 1; i < size; i++) { - long zigzagEncoded = encodedData[i]; - // ZigZag解码: (n >>> 1) ^ (-(n & 1)) - long diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); - result[i] = prev + diff; - prev = result[i]; - } - - return result; - } - private static class BitWriter { - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - private long acc = 0L; // holds currently buffered bits (lowest "accBits" bits are valid) - private int accBits = 0; // number of bits in acc - - // writeBits expects bits in LSB-aligned form (i.e., "masked" value). We append MSB-first as: acc = (acc << bitCount) | bits - void writeBits(long bits, int bitCount) { - if (bitCount == 0) return; - if (bitCount == 64) { - // split into two 32-bit writes to avoid shifting by 64 - writeBits((bits >>> 32) & 0xFFFFFFFFL, 32); - writeBits(bits & 0xFFFFFFFFL, 32); - return; - } - long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); - long v = bits & mask; - acc = (acc << bitCount) | v; - accBits += bitCount; - while (accBits >= 8) { - int shift = accBits - 8; - int outb = (int) ((acc >>> shift) & 0xFFL); - out.write(outb); - if (shift > 0) { - acc &= ((1L << shift) - 1L); - } else { - acc = 0L; - } - accBits = shift; - } - } - - byte[] finish() { - if (accBits > 0) { - int outb = (int) ((acc << (8 - accBits)) & 0xFFL); - out.write(outb); - acc = 0L; - accBits = 0; - } - return out.toByteArray(); - } - } - // private static class BitReader { -// final byte[] data; -// private long acc = 0L; -// private int accBits = 0; -// private int idx = 0; -// -// BitReader(byte[] data) { -// this.data = data; -// } -// -// long readBits(int bitCount) throws IOException { -// if (bitCount == 0) return 0L; -// while (accBits < bitCount) { -// if (idx < data.length) { -// acc = (acc << 8) | (data[idx++] & 0xFFL); -// accBits += 8; -// } else { -// // pad with zeros if stream ends prematurely -// acc = (acc << (bitCount - accBits)); -// accBits = bitCount; -// } -// } -// int shift = accBits - bitCount; -// long mask = (bitCount == 64) ? ~0L : ((1L << bitCount) - 1L); -// long v = (acc >>> shift) & mask; -// if (shift > 0) { -// acc &= ((1L << shift) - 1L); -// } else { -// acc = 0L; -// } -// accBits = shift; -// return v; -// } -// } - public static final class BitReader { - private final byte[] data; - private int bitPos; // global bit position from start of data[] - - public BitReader(byte[] data) { - this(data, 0); - } - - public BitReader(byte[] data, int byteOffset) { - this.data = data; - this.bitPos = byteOffset * 8; - } - - /** - * Read n bits (0 <= n <= 64), return as unsigned long. - */ - public long readBits(int n) { - if (n == 0) return 0L; - if (n < 0 || n > 64) { - throw new IllegalArgumentException("n must be between 0 and 64"); - } - - long result = 0L; - int bitsRemaining = n; - - while (bitsRemaining > 0) { - int byteIndex = bitPos >>> 3; // current byte - int bitOffset = bitPos & 7; // offset inside byte [0..7] - - // 添加边界检查 - if (byteIndex >= data.length) { - // 如果已经超出数据范围，填充0并返回 - result = (result << bitsRemaining); - bitPos += bitsRemaining; - return result; - } - - int bitsFromCurrentByte = Math.min(8 - bitOffset, bitsRemaining); - - // Load byte as unsigned - int curByte = data[byteIndex] & 0xFF; - - // Shift to get the relevant bits - int shift = 8 - bitOffset - bitsFromCurrentByte; - int chunk = (curByte >>> shift) & ((1 << bitsFromCurrentByte) - 1); - - result = (result << bitsFromCurrentByte) | chunk; - - bitPos += bitsFromCurrentByte; - bitsRemaining -= bitsFromCurrentByte; - } - - return result; - } - - /** - * @return total bits consumed since creation / since byteOffset - */ - public int consumedBits() { - return bitPos; - } - - /** - * @return current bit position (alias) - */ - public int bitPosition() { - return bitPos; - } - - /** - * @return remaining bits available for reading - */ - public int remainingBits() { - return (data.length * 8) - bitPos; - } - } - private static byte[] performBitPackingCompression64_fast(long[] dataArray, List packs, int pack_size, int originalLength) throws IOException { - // 计算一些元信息 - int totalPacks = packs.size(); - int maxOctadsInAnyPack = 0; - for ( Pack p : packs) if (p.size > maxOctadsInAnyPack) maxOctadsInAnyPack = p.size; - - // bits needed to encode counts in range [0..maxOctadsInAnyPack] - int bitsForCount = 1; - while ((1L << bitsForCount) <= maxOctadsInAnyPack) bitsForCount++; - if (bitsForCount <= 0) bitsForCount = 1; - - // 准备输出缓冲（先写 header 的整数字段，meta/data 用 BitWriter 位流） - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - - dos.writeByte(totalPacks); - dos.writeByte(bitsForCount); // 1 byte is enough to carry this small number - dos.flush(); - - // --- Meta bitstream --- - BitWriter metaWriter = new BitWriter(); - - // For each pack: write pack.size using bitsForCount bits, then for each octad write its bitWidth using 6 bits - for ( Pack pack : packs) { - // write octad count - metaWriter.writeBits(pack.size, bitsForCount); - metaWriter.writeBits(pack.bitWidths.get(0), 6); -// // write each octad's bitWidth using 6 bits. -// // NOTE: we map bitWidth==64 -> store 63 (as sentinel). 解码端须按此约定把 63 映射回 64。 -// for (int i = 0; i < pack.size; ++i) { -// int bw = pack.bitWidths.get(i); -// int store = bw; -// if (bw == 64) store = 63; -// if (store < 0) store = 0; -// if (store > 63) store = 63; // safety clamp -// metaWriter.writeBits(store, 5); -// } - } - - byte[] metaBytes = metaWriter.finish(); - dos.write(metaBytes); - dos.flush(); - - // --- Data bitstream --- - BitWriter dataWriter = new BitWriter(); - - // For each pack, find packMaxBitWidth and write each group's pack_size values using packMaxBitWidth bits - for ( Pack pack : packs) { - int packMaxBW = pack.maxBitWidth; - // no change for packMaxBW == 64: BitWriter supports splitting 64 into two 32-bit writes - for (int i = 0; i < pack.size; ++i) { - int originalGroupIndex = pack.indices.get(0); - int startPos = originalGroupIndex * pack_size; - for (int j = 0; j < pack_size; ++j) { - long val; -// if (startPos + j < dataArray.length) { - val = dataArray[startPos + j]; -// } else { -// val = 0L; -// } - // mask value to packMaxBW bits (if packMaxBW == 64, mask preserves full 64 bits) - long mask; - if (packMaxBW == 0) { - dataWriter.writeBits(0L, 0); // nothing to write - } else { - if (packMaxBW == 64) { - // write full 64-bit value (BitWriter handles split) - dataWriter.writeBits(val, 64); - } else { - mask = (1L << packMaxBW) - 1L; - long masked = val & mask; - dataWriter.writeBits(masked, packMaxBW); - } - } - } - } - } - - byte[] dataBytes = dataWriter.finish(); - dos.write(dataBytes); - dos.flush(); - - return baos.toByteArray(); - } - - // ========== packOctads (updated to accept originalLength for compression) ========== - static PackingResult packOctads(List bitWidths, RLDecisionModel model, List< DecisionPoint> decisionTrace, int pack_size, long[] dataArray, int originalLength) { - PackingResult result = new PackingResult(); - Pack currentPack = new Pack(); - int globalMaxLog = 0; - int packCount = 0; - - Random localRng = ThreadLocalRandom.current(); - - for (int i = 0; i < bitWidths.size(); ++i) { - int b = bitWidths.get(i); - - if (currentPack.size == 0) { - currentPack.addOctad(i, b); - } else if (b == currentPack.maxBitWidth) { - currentPack.addOctad(i, b); - } else { - float[] feat = new float[INPUT_DIM]; - feat[0] = currentPack.size / 100.0f; - feat[1] = currentPack.maxBitWidth / 64.0f; - feat[2] = b / 64.0f; - feat[3] = packCount / 100.0f; - feat[4] = globalMaxLog / 10.0f; - - float probability = model.forwardProb(feat); - - boolean shouldMerge; - if (localRng.nextFloat() < model.explorationRate) { - shouldMerge = (localRng.nextFloat() > 0.5f); - } else { - shouldMerge = probability > 0.5f; - } - - if (decisionTrace != null) { - decisionTrace.add(new DecisionPoint(currentPack.size, currentPack.maxBitWidth, b, packCount, globalMaxLog, shouldMerge, probability)); - } - - if (shouldMerge) { - currentPack.addOctad(i, b); - } else { - result.dataCostA += currentPack.dataCost(pack_size); - int logSize = currentPack.logSize(); - if (logSize > globalMaxLog) globalMaxLog = logSize; - result.packs.add(currentPack); - packCount++; - - currentPack = new Pack(); - currentPack.addOctad(i, b); - } - } - } - - if (currentPack.size > 0) { - result.dataCostA += currentPack.dataCost(pack_size); - int logSize = currentPack.logSize(); - if (logSize > globalMaxLog) globalMaxLog = logSize; - result.packs.add(currentPack); - packCount++; - } - - result.packCount = packCount; - result.calculateCost(globalMaxLog); - - // 执行实际的bitpacking压缩（如果提供了 dataArray） - if (dataArray != null) { - try { - result.compressedData = performBitPackingCompression64_fast(dataArray, result.packs, pack_size, originalLength); - } catch (IOException e) { - System.err.println("Compression failed: " + e.getMessage()); - result.compressedData = null; - } - } - - return result; - } - -// public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { -// try { -// // 这里需要根据实际的压缩格式来解析 -// // 假设compressedData包含bit-packed数据 -// ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); -// DataInputStream dis = new DataInputStream(bais); -// -// // 读取bit-packed数据 -// int totalGroups = bitWidths.length; -// long[] result = new long[totalGroups * packSize]; -// int resultIndex = 0; -// -// BitReader reader = new BitReader(compressedData); -// -// for (int g = 0; g < totalGroups; ++g) { -// int bw = bitWidths[g]; -// for (int k = 0; k < packSize; ++k) { -// if (bw == 0) { -// result[resultIndex++] = 0L; -// } else if (bw == 64) { -// long high = reader.readBits(32); -// long low = reader.readBits(32); -// long v = (high << 32) | (low & 0xFFFFFFFFL); -// result[resultIndex++] = v; -// } else { -// long v = reader.readBits(bw); -// result[resultIndex++] = v; -// } -// } -// } -// -// // 只取原始长度的数据并Sprintz解码 -// return Arrays.copyOf(result, originalLength); -// -// } catch (IOException e) { -// System.err.println("Fast decompression failed: " + e.getMessage()); -// return new long[0]; -// } -// } - - public static long[] fastDecompress(byte[] compressedData, int[] bitWidths, int packSize, int originalLength) { - if (compressedData == null || compressedData.length == 0) { - System.err.println("Compressed data is null or empty"); - return new long[0]; - } - - try { - ByteArrayInputStream bais = new ByteArrayInputStream(compressedData); - DataInputStream dis = new DataInputStream(bais); - - // === Header === - int totalPacks = dis.readUnsignedByte(); - int bitsForCount = dis.readUnsignedByte(); - - // === Read meta bitstream === - int metaStartOffset = 2; // two bytes read - BitReader metaReader = new BitReader(compressedData, metaStartOffset); - - // 解析每个pack的信息 - List packInfos = new ArrayList<>(); - int totalValuesToDecode = 0; - - for (int p = 0; p < totalPacks; ++p) { - int octadCount = (int) metaReader.readBits(bitsForCount); - int packBitWidth = (int) metaReader.readBits(6); - if (packBitWidth == 63) packBitWidth = 64; - - packInfos.add(new PackInfo(octadCount, packBitWidth)); - totalValuesToDecode += octadCount * packSize; - } - - // 计算数据部分的起始位置 - int metaBitsUsed = metaReader.consumedBits(); - int dataStartByte = metaStartOffset + (metaBitsUsed + 7) / 8; - - // 检查数据起始位置是否超出压缩数据范围 - if (dataStartByte >= compressedData.length) { -// System.err.println("Data start position exceeds compressed data length"); - return new long[0]; - } - - // === Data bitstream === - BitReader dataReader = new BitReader(compressedData, dataStartByte); - List resultList = new ArrayList<>(); - - // === 按pack解码数据 === - for ( PackInfo packInfo : packInfos) { - int octadCount = packInfo.octadCount; - int bitWidth = packInfo.bitWidth; - - // 检查剩余数据是否足够 - if (dataReader.remainingBits() < (long) octadCount * packSize * bitWidth) { -// System.err.println("Insufficient data for decoding pack. Expected: " + -// (octadCount * packSize * bitWidth) + " bits, Available: " + -// dataReader.remainingBits() + " bits"); - break; - } - - // 每个octad包含packSize个值 - for (int i = 0; i < octadCount; ++i) { - for (int j = 0; j < packSize; ++j) { - long value; - if (bitWidth == 0) { - value = 0L; - } else if (bitWidth == 64) { - // 64位特殊处理：分成两个32位读取 - long high = dataReader.readBits(32); - long low = dataReader.readBits(32); - value = (high << 32) | low; - } else { - value = dataReader.readBits(bitWidth); - } - resultList.add(value); - } - } - } - - // 转换为数组并截取到原始长度 - long[] result = new long[Math.min(resultList.size(), originalLength)]; - for (int i = 0; i < result.length; i++) { - result[i] = resultList.get(i); - } - -// System.out.println("Decompression completed: " + result.length + " values decoded"); - return result; - - } catch (Exception e) { - System.err.println("Fast decompression failed: " + e.getMessage()); - e.printStackTrace(); - return new long[0]; - } - } - - // 辅助类，存储pack信息 - static class PackInfo { - int octadCount; - int bitWidth; - - PackInfo(int octadCount, int bitWidth) { - this.octadCount = octadCount; - this.bitWidth = bitWidth; - } - } - // ========== Training loop (trainModel) ========== - static RLDecisionModel trainModel(int epochs, String csvFilePath) { - System.err.println("Training RL model from CSV data..."); - RLDecisionModel model = new RLDecisionModel(); - List> sequences = loadDataFromCSV(csvFilePath); - if (sequences.isEmpty()) { - System.err.println("No data loaded from CSV. Returning initial model."); - return model; - } - System.err.println("Loaded " + sequences.size() + " sequences from CSV"); - - List decisionTrace = new ArrayList<>(); - for (int epoch = 1; epoch <= epochs; ++epoch) { - long startTime = System.nanoTime(); - float totalReward = 0.0f; - float totalLoss = 0.0f; - int processedSequences = 0; - - for (List bitWidths : sequences) { - decisionTrace.clear(); - // training does not perform actual compression, pass dataArray=null and originalLength=0 - PackingResult result = packOctads(bitWidths, model, decisionTrace, 8, null, 0); - - float reward = (float) result.totalCost / 500000.0f; - totalReward += reward; - - float loss = model.train(decisionTrace, reward); - totalLoss += loss; - processedSequences++; - } - - long durationMs = (System.nanoTime() - startTime) / 1_000_000L; - - if (epoch % 10 == 0 || epoch == 1 || epoch == epochs) { - System.out.printf("Epoch %d: Avg Reward = %.6f, Avg Loss = %.6f, Time = %d ms%n", - epoch, - totalReward / processedSequences, - totalLoss / processedSequences, - durationMs); - } else { - System.out.printf("Epoch %d done. Time = %d ms%n", epoch, durationMs); - } - } - - return model; - } - // ========== performanceTest (optimized internals) ========== - static void performanceTestPack8(RLDecisionModel model, String directory, String outputDirStr) { - System.out.println("\nPerformance Testing..."); - Path outdir = Paths.get(outputDirStr); - try { - if (!Files.exists(outdir)) Files.createDirectories(outdir); - } catch (IOException e) { - System.err.println("Cannot create output dir: " + outputDirStr); - return; - } - - try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { - for (Path entry : ds) { - if (!Files.isRegularFile(entry)) continue; - String fname = entry.getFileName().toString(); - if (IGNORE_FILES.contains(fname)) continue; - - System.out.println("Processing " + fname + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - - try (BufferedReader br = Files.newBufferedReader(entry)) { - String line; - while ((line = br.readLine()) != null) { - String[] tokens = line.split(","); - for (String token : tokens) { - String t = trimStr(token); - if (!t.isEmpty()) { - numbers.add(t); - int dec = 0; - int pos = t.indexOf('.'); - if (pos != -1) dec = t.length() - pos - 1; - decimalPlaces.add(dec); - } - } - } - } catch (IOException e) { - System.err.println("Cannot open " + entry.toString()); - continue; - } - - if (numbers.isEmpty()) continue; - - Path outPath = outdir.resolve(fname); - try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { - writer.write("Input Direction,Encoding Algorithm,Encoding Time,Decoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); - - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - long modelCost = 0; - long modelTime = 0; - long compressedSize = 0; - long modelDecodeTime = 0; - - for (int rep = 0; rep < time_of_repeat; ++rep) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(numbers.size(), i + CHUNK_SIZE); - if (end - i <= 2) continue; - List chunkNumbers = numbers.subList(i, end); - int decimalMax = 0; - for (int k = i; k < end; ++k) { - if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); - } - - long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - long[] scaledInts = sprintz(scaledInt); - - int remainder = scaledInts.length % pack_size; - int padding = (remainder == 0) ? 0 : pack_size - remainder; - long[] padded = new long[scaledInts.length + padding]; - System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); - if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); - - int groups = padded.length / pack_size; - int[] bitWidths = new int[groups]; - int gidx = 0; - for (int si = 0; si < padded.length; si += pack_size) { - long maxInGroup = 0; - for (int sj = si; sj < si + pack_size; ++sj) { - long v = padded[sj]; - if (v > maxInGroup) maxInGroup = v; - } - int bitWidth = 0; - if (maxInGroup > 0) { - bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - } else { - bitWidth = 0; - } - bitWidths[gidx++] = bitWidth; - } - - // pass original length (un-padded) so decoder can trim - List bitWidthsList = new ArrayList<>(groups); - for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); - - PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); - long duration = System.nanoTime() - startTime; - - - // 使用快速解压 - if (res.compressedData != null) { - long decodeStartTime = System.nanoTime(); - long[] decompressed = fastDecompress(res.compressedData, bitWidths, pack_size, scaledInts.length); - long[] decompressed_final = sprintzDecode(decompressed); - long decodeDuration = System.nanoTime() - decodeStartTime; - modelDecodeTime += decodeDuration; -// System.out.println(decodeDuration); - } - - modelTime += duration; - modelCost += (res.compressedData.length* 8L); - - } - } - - modelCost /= time_of_repeat; - modelTime /= time_of_repeat; - modelDecodeTime /= time_of_repeat; - - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms - double decodeThroughput = (double) (numbers.size() * 8000) / modelDecodeTime; // points per second - writer.write(entry.toString() + ","); - writer.write("SPRINTZ-RL,"); - writer.write(String.valueOf(modelTime_throughput) + ","); - writer.write(String.valueOf(decodeThroughput) + ","); - writer.write(String.valueOf(numbers.size()) + ","); - writer.write(String.valueOf(modelCost) + ","); - writer.write(String.valueOf(pack_size) + ","); - writer.write(String.valueOf(model_ratio) + "\n"); - } - } catch (IOException e) { - System.err.println("Error writing output file for " + fname); - } - } - } catch (IOException e) { - System.err.println("Error iterating directory: " + directory); - } - } - - public static void main(String[] args) { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv";// args.length > 0 ? args[0] : ""; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel";//args.length > 1 ? args[1] : ""; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_rl";// args.length > 2 ? args[2] : "./output_BPRL"; - - int epochs = 20; - - if (args.length >= 1) trainCsv = args[0]; - if (args.length >= 2) dataDir = args[1]; - if (args.length >= 3) outDir = args[2]; - - RLDecisionModel model = new RLDecisionModel(); - if (!trainCsv.isEmpty()) { - model = trainModel(epochs, trainCsv); - } else { - System.err.println("No training CSV given. Using randomly initialized RL model."); - } - - if (!dataDir.isEmpty()) { - performanceTestPack8(model, dataDir, outDir); - } else { - System.err.println("No data directory provided for performanceTest. Exiting."); - } - } - - - // ========== performanceTest (optimized internals) ========== - static void performanceTest(RLDecisionModel model, String directory, String outputDirStr) { - System.out.println("\nPerformance Testing..."); - Path outdir = Paths.get(outputDirStr); - try { - if (!Files.exists(outdir)) Files.createDirectories(outdir); - } catch (IOException e) { - System.err.println("Cannot create output dir: " + outputDirStr); - return; - } - - try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { - for (Path entry : ds) { - if (!Files.isRegularFile(entry)) continue; - String fname = entry.getFileName().toString(); - if (IGNORE_FILES.contains(fname)) continue; - - System.out.println("Processing " + fname + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - - try (BufferedReader br = Files.newBufferedReader(entry)) { - String line; - while ((line = br.readLine()) != null) { - String[] tokens = line.split(","); - for (String token : tokens) { - String t = trimStr(token); - if (!t.isEmpty()) { - numbers.add(t); - int dec = 0; - int pos = t.indexOf('.'); - if (pos != -1) dec = t.length() - pos - 1; - decimalPlaces.add(dec); - } - } - } - } catch (IOException e) { - System.err.println("Cannot open " + entry.toString()); - continue; - } - - if (numbers.isEmpty()) continue; - - Path outPath = outdir.resolve(fname); - try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { - writer.write("Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); - - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 9; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - System.out.println(pack_size); - long modelCost = 0; - long modelTime = 0; - long compressedSize = 0; - - for (int rep = 0; rep < time_of_repeat; ++rep) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(numbers.size(), i + CHUNK_SIZE); - if (end - i <= 2) continue; - List chunkNumbers = numbers.subList(i, end); - int decimalMax = 0; - for (int k = i; k < end; ++k) { - if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); - } - - long[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - long[] scaledInts = sprintz(scaledInt); - - int remainder = scaledInts.length % pack_size; - int padding = (remainder == 0) ? 0 : pack_size - remainder; - long[] padded = new long[scaledInts.length + padding]; - System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); - if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); - - int groups = padded.length / pack_size; - int[] bitWidths = new int[groups]; - int gidx = 0; - for (int si = 0; si < padded.length; si += pack_size) { - long maxInGroup = 0; - for (int sj = si; sj < si + pack_size; ++sj) { - long v = padded[sj]; - if (v > maxInGroup) maxInGroup = v; - } - int bitWidth = 0; - if (maxInGroup > 0) { - bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - } else { - bitWidth = 0; - } - bitWidths[gidx++] = bitWidth; - } - - // pass original length (un-padded) so decoder can trim - List bitWidthsList = new ArrayList<>(groups); - for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); - - PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += (res.compressedData.length*8); - -// if (rep == 0) { -// compressedSize += (res.compressedData != null) ? res.compressedData.length : 0; -// } - } - } - - modelCost /= time_of_repeat; - modelTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms - - writer.write(entry.toString() + ","); - writer.write("SPRINTZ-RL,"); - writer.write(String.valueOf(modelTime_throughput) + ","); - writer.write(String.valueOf(numbers.size()) + ","); - writer.write(String.valueOf(modelCost) + ","); - writer.write(String.valueOf(pack_size) + ","); - writer.write(String.valueOf(model_ratio) + "\n"); - } - } catch (IOException e) { - System.err.println("Error writing output file for " + fname); - } - } - } catch (IOException e) { - System.err.println("Error iterating directory: " + directory); - } - } - @Test - public void TestVarPackSize() { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv";// args.length > 0 ? args[0] : ""; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel";//args.length > 1 ? args[1] : ""; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_RL_vary_pack_size";// args.length > 2 ? args[2] : "./output_BPRL"; - - int epochs = 20; - - RLDecisionModel model = new RLDecisionModel(); - model = trainModel(epochs, trainCsv); - performanceTest(model, dataDir, outDir); - } - - - static void performanceTestVariableChunkSize(RLDecisionModel model, String directory, String outputDirStr) { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - Path outdir = Paths.get(outputDirStr); - try { - if (!Files.exists(outdir)) Files.createDirectories(outdir); - } catch (IOException e) { - System.err.println("Cannot create output dir: " + outputDirStr); - return; - } - - // Define the chunk sizes to test (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - try (DirectoryStream ds = Files.newDirectoryStream(Paths.get(directory))) { - for (Path entry : ds) { - if (!Files.isRegularFile(entry)) continue; - String fname = entry.getFileName().toString(); - if (IGNORE_FILES.contains(fname)) continue; - - System.out.println("Processing " + fname + " with variable chunk sizes..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - - try (BufferedReader br = Files.newBufferedReader(entry)) { - String line; - while ((line = br.readLine()) != null) { - String[] tokens = line.split(","); - for (String token : tokens) { - String t = trimStr(token); - if (!t.isEmpty()) { - numbers.add(t); - int dec = 0; - int pos = t.indexOf('.'); - if (pos != -1) dec = t.length() - pos - 1; - decimalPlaces.add(dec); - } - } - } - } catch (IOException e) { - System.err.println("Cannot open " + entry.toString()); - continue; - } - - if (numbers.isEmpty()) continue; - - Path outPath = outdir.resolve(fname.replace(".", "_chunksize_test.")); - try (BufferedWriter writer = Files.newBufferedWriter(outPath)) { - writer.write("m,Input Direction,Encoding Algorithm,Encoding Time,Points,Compressed Size,Pack Size,Compression Ratio\n"); - - int time_of_repeat = 50; // Reduced for faster testing with multiple chunk sizes - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - long[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - long[] scaledInts_all = new long[totalLength]; - - int currentIndex = 0; - for (long[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - - // Test each chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for (int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - long modelCost = 0; - long modelTime = 0; - long compressedSize = 0; - - for (int rep = 0; rep < time_of_repeat; ++rep) { - for (int i = 0; i < numbers.size(); i += chunkSize) { -// int end = Math.min(numbers.size(), i + chunkSize); -// if (end - i <= 2) continue; -// List chunkNumbers = numbers.subList(i, end); -// int decimalMax = 0; -// for (int k = i; k < end; ++k) { -// if (decimalPlaces.get(k) > decimalMax) decimalMax = decimalPlaces.get(k); -// } -// -// long[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - int end = Math.min(i + chunkSize, numbers.size()); - long[] scaledInt = new long[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - long[] scaledInts = sprintz(scaledInt); - - int remainder = scaledInts.length % pack_size; - int padding = (remainder == 0) ? 0 : pack_size - remainder; - long[] padded = new long[scaledInts.length + padding]; - System.arraycopy(scaledInts, 0, padded, 0, scaledInts.length); - if (padding > 0) Arrays.fill(padded, scaledInts.length, padded.length, 0L); - - int groups = padded.length / pack_size; - int[] bitWidths = new int[groups]; - int gidx = 0; - for (int si = 0; si < padded.length; si += pack_size) { - long maxInGroup = 0; - for (int sj = si; sj < si + pack_size; ++sj) { - long v = padded[sj]; - if (v > maxInGroup) maxInGroup = v; - } - int bitWidth = 0; - if (maxInGroup > 0) { - bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - } else { - bitWidth = 0; - } - bitWidths[gidx++] = bitWidth; - } - - // pass original length (un-padded) so decoder can trim - List bitWidthsList = new ArrayList<>(groups); - for (int x = 0; x < groups; ++x) bitWidthsList.add(bitWidths[x]); - - PackingResult res = packOctads(bitWidthsList, model, null, pack_size, padded, scaledInts.length); - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += (res.compressedData.length* 8L); - - } - } - - modelCost /= time_of_repeat; - modelTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); // compressed / original bytes - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) modelTime; // points/ms - - writer.write(String.valueOf(chunkSize/8) + ","); - writer.write(entry.toString() + ","); - writer.write("sprintz-RL,"); - writer.write(String.valueOf(modelTime_throughput) + ","); - writer.write(String.valueOf(numbers.size()) + ","); - writer.write(String.valueOf(modelCost) + ","); - writer.write(String.valueOf(pack_size) + ","); - writer.write(String.valueOf(model_ratio) + "\n"); - } - } - } catch (IOException e) { - System.err.println("Error writing output file for " + fname); - } - } - } catch (IOException e) { - System.err.println("Error iterating directory: " + directory); - } - } - @Test - public void TestVariableChunkSize() { - String trainCsv = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String dataDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprint_RL_vary_m"; - - int epochs = 20; - - RLDecisionModel model = new RLDecisionModel(); - model = trainModel(epochs, trainCsv); - performanceTestVariableChunkSize(model, dataDir, outDir); - } - -} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java deleted file mode 100644 index bd1c8457f..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FBitpacking512.java +++ /dev/null @@ -1,798 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - - -public class FBitpacking512 { - - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - private static final int CHUNK_SIZE = 1024; - - public static int getBitWith(int num) { - if (num == 0) - return 1; - else - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static int getCount(long long1, int mask) { - return ((int) (long1 & mask)); - } - - public static int getUniqueValue(long long1, int left_shift) { - return ((int) ((long1) >> left_shift)); - } - - public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer); - } - - private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static int bytes2Integer(byte[] encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded[i + start] & 0xFF; - value |= b; - } - return value; - } - - private static long bytesLong2Integer(byte[] encoded, int decode_pos) { - long value = 0; - for (int i = 0; i < 4; i++) { - value <<= 8; - int b = encoded[i + decode_pos] & 0xFF; - value |= b; - } - return value; - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - // total bits which have read from 'buf' to 'buffer'. i.e., - // number of available bits to be decoded. - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - // If current available bits are not enough to decode one Integer, - // then add next byte from buf to 'buffer' until totalBits >= width - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - // If current available bits are enough to decode one Integer, - // then decode one Integer one by one until left bits in 'buffer' is - // not enough to decode one Integer. - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - - return encode_pos; - - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - // 新增的解压函数 -// public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { -// List result = new ArrayList<>(); -// int decodePos = 0; -// -// for (int group = 0; group < bitWidths.length; group++) { -// int bitWidth = compressedData[decodePos++] & 0xFF; -// -// // 解压当前分组 -// ArrayList groupData = new ArrayList<>(); -// unpack8Values(compressedData, decodePos, bitWidth, groupData); -// -// // 添加解压出的数据 -// for (int i = 0; i < pack_size; i++) { -// if (result.size() < originalLength) { -// result.add(groupData.get(i)); -// } -// } -// -// decodePos += bitWidth; -// } -// -// // 转换为数组返回 -// int[] decodedArray = new int[result.size()]; -// for (int i = 0; i < result.size(); i++) { -// decodedArray[i] = result.get(i); -// } -// return decodedArray; -// } - - public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { - int[] result = new int[originalLength]; // 直接使用数组 - int resultIndex = 0; - int decodePos = 0; - - for (int group = 0; group < bitWidths.length && resultIndex < originalLength; group++) { - int bitWidth = compressedData[decodePos++] & 0xFF; - - // 预分配固定大小的列表 - ArrayList groupData = new ArrayList<>(pack_size); - unpack8Values(compressedData, decodePos, bitWidth, groupData); - - // 批量拷贝 - int copyLength = Math.min(pack_size, originalLength - resultIndex); - for (int i = 0; i < copyLength; i++) { - result[resultIndex++] = groupData.get(i); - } - - decodePos += bitWidth; - } - - return result; - } - - // 新增的完整解压函数（包含头部信息解析） - public static int[] decodeBitPackingWithHeader(byte[] encodedWithHeader, int pack_size) { - // 解析头部信息 - 假设前4个字节存储原始数据长度 - int originalLength = bytes2Integer(encodedWithHeader, 0, 4); - - // 解析分组数 - int groupCount = bytes2Integer(encodedWithHeader, 4, 4); - - // 解析位宽数组 - int[] bitWidths = new int[groupCount]; - int headerSize = 8; // 4字节原始长度 + 4字节分组数 - for (int i = 0; i < groupCount; i++) { - bitWidths[i] = encodedWithHeader[headerSize + i] & 0xFF; - } - - // 压缩数据起始位置 - int dataStart = headerSize + groupCount; - byte[] compressedData = new byte[encodedWithHeader.length - dataStart]; - System.arraycopy(encodedWithHeader, dataStart, compressedData, 0, compressedData.length); - - // 调用解压函数 - return decodeBitPacking(compressedData, bitWidths, pack_size, originalLength); - } - - private static int[] scaleNumbers(List numbers, int decimalMax) { - // 1. 预先计算缩放因子 - BigDecimal scale = BigDecimal.TEN.pow(decimalMax); - int size = numbers.size(); - int[] result = new int[size]; - - if (size == 0) { - return result; - } - - // 2. 单次遍历完成所有转换和最小值查找 - BigDecimal min = null; - BigDecimal[] scaledValues = new BigDecimal[size]; - - for (int i = 0; i < size; i++) { - BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); - scaledValues[i] = val; - if (min == null || val.compareTo(min) < 0) { - min = val; - } - } - - // 3. 处理第一个元素 - BigDecimal first = scaledValues[0].subtract(min); - result[0] = first.toBigInteger().intValue(); - - // 4. 处理后续元素（差分+ZigZag） - for (int i = 1; i < size; i++) { - BigDecimal current = scaledValues[i].subtract(min); - result[i]=current.toBigInteger().intValue(); - } - - return result; - } - - public static void main(String[] args) throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - // 更新表头，增加解压吞吐率列 - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 500; - - - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; // 新增：解压时间 - - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - // 1. 找出当前8个元素中的最大值 - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / 8] = bitWidth; - } - -// int fixed_block = CHUNK_SIZE/40; - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, 8); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 新增：测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPacking(compressedData, bitWidths, 8, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - -// // 可选：验证解压数据的正确性（只在第一次重复时验证） -// if (j == 0) { -// boolean correct = true; -// for (int k = 0; k < scaledInts.length; k++) { -// if (scaledInts[k] != decodedData[k]) { -// correct = false; -// System.err.println("Decompression error at index " + k + -// ": expected " + scaledInts[k] + ", got " + decodedData[k]); -// break; -// } -// } -//// if (correct) { -//// System.out.println("Decompression verified successfully for chunk " + (i/CHUNK_SIZE)); -//// } -// } - } - - } - modelCost = modelCost/time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - modelDecodeTime = (modelDecodeTime)/time_of_repeat; // 平均解压时间 - - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000L)/ (double) (modelTime); - double modelDecodeTime_throughput = (double)(numbers.size()*8000L)/ (double) (modelDecodeTime); - - // 更新输出记录，包含解压吞吐率 - String[] record = { - file.toString(), - "BP", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); - - System.out.println("Encoding throughput: " + modelTime_throughput + " MB/s"); - System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); - System.out.println("Compression ratio: " + model_ratio); - } - - } - - public static byte[] encodeBitPacking(int[] paddedArray, int[] bitWidths, int pack_size) { - List result = new ArrayList<>(); - - int totalGroups = bitWidths.length; - int max_bit_width = 0; - - for (int i = 0; i < totalGroups; i++) { - if (bitWidths[i] > max_bit_width) { - max_bit_width = bitWidths[i]; - } - } - int totalBitPackedBytes = (max_bit_width*pack_size*totalGroups+7)/8; - byte[] bitPackedData = new byte[totalBitPackedBytes +totalGroups+ 32]; - int encodePos = 0; - - for (int group = 0; group < totalGroups; group++) { - int startIndex = group * pack_size; - ArrayList groupData = new ArrayList<>(); - for (int i = 0; i < pack_size; i++) { - if (startIndex + i < paddedArray.length) { - groupData.add(paddedArray[startIndex + i]); - } else { - groupData.add(0); - } - } - - bitPackedData[encodePos++] = (byte) bitWidths[group]; - encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); - } - - for (int i = 0; i < encodePos; i++) { - result.add(bitPackedData[i]); - } - - byte[] finalResult = new byte[result.size()]; - for (int i = 0; i < result.size(); i++) { - finalResult[i] = result.get(i); - } - - return finalResult; - } - - public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { - int blocksize= bitWidths.length; - int totalCost = 0; - int numPacks = (int) Math.ceil((double) blocksize / fixed_pack); - - for (int pack = 0; pack < numPacks; pack++) { - int start = pack * fixed_pack; - int end = Math.min(start + fixed_pack, blocksize); - - int maxBitWidth = 0; - for (int i = start; i < end; i++) { - if (bitWidths[i] > maxBitWidth) { - maxBitWidth = bitWidths[i]; - } - } - - totalCost += pack_size * (end-start) * maxBitWidth; - } - - totalCost += 5 * blocksize / fixed_pack; - return totalCost; - } - - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP_vary_pack_size"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - // 方法：强化学习 - int modelCost = 0; - long modelTime = 0; - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); - int cur_cost = compressedData.length * 8; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - String[] record = { - file.toString(), - "BP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - writer.close(); - } - - } - - // 新增方法：测试不同chunk size的表现 - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BP_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 -// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); -// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); - - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - -// 分批处理，每1024个元素一批 - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - int[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - - int currentIndex = 0; - for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); -// System.out.println(numbers.subList(0,1000)); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); - -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); - - int end = Math.min(i + chunkSize, numbers.size()); - int[] scaledInts = new int[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为pack_size的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; -// if(i==0){ -// System.out.println(Arrays.toString(paddedArray)); -// } - - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; -// System.out.println(bitWidth); - } - - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); - int cur_cost = compressedData.length * 8; - long duration = System.nanoTime() - startTime; - - modelTime += (duration); - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "BP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); -// break; - } - } -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java deleted file mode 100644 index 7280298ae..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/FSprintz512.java +++ /dev/null @@ -1,1158 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -//import org.openjdk.jol.info.ClassLayout; -//import org.openjdk.jol.info.GraphLayout; - -public class FSprintz512 { - private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv", - "POI-lon.csv","Air-sensor.csv","Basel-wind.csv","Basel-temp.csv"); -// private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv"); - private static final int CHUNK_SIZE = 1024; - // 轻量级Octad表示 - - public static int getBitWith(int num) { - if (num == 0) - return 1; - else - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static int getCount(long long1, int mask) { - return ((int) (long1 & mask)); - } - - public static int getUniqueValue(long long1, int left_shift) { - return ((int) ((long1) >> left_shift)); - } - - public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer); - } - - private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static int bytes2Integer(byte[] encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded[i + start] & 0xFF; - value |= b; - } - return value; - } - - private static long bytesLong2Integer(byte[] encoded, int decode_pos) { - long value = 0; - for (int i = 0; i < 4; i++) { - value <<= 8; - int b = encoded[i + decode_pos] & 0xFF; - value |= b; - } - return value; - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - // total bits which have read from 'buf' to 'buffer'. i.e., - // number of available bits to be decoded. - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - // If current available bits are not enough to decode one Integer, - // then add next byte from buf to 'buffer' until totalBits >= width - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - // If current available bits are enough to decode one Integer, - // then decode one Integer one by one until left bits in 'buffer' is - // not enough to decode one Integer. - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - - return encode_pos; - - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - private static int zigzagEncode(int n) { - return (n << 1) ^ (n >> 31); - } - public static int[] scaleNumbers(List numbers, int decimalMax) { - int scale = (int) Math.pow(10, decimalMax); - int size = numbers.size(); - int[] result = new int[size]; - - if (size == 0) { - return result; - } - - // 1. Parse all numbers and scale them up - int[] scaledValues = new int[size]; - for (int i = 0; i < size; i++) { - String numStr = numbers.get(i); - // Parse the number (handling both "123.456" and "123" cases) - String[] parts = numStr.split("\\."); - int whole = Integer.parseInt(parts[0]); - - // Handle fractional part - int fraction = 0; - if (parts.length > 1) { - String fractionStr = parts[1]; - // Pad with zeros if necessary to ensure proper scaling - if (fractionStr.length() < decimalMax) { - while (fractionStr.length() < decimalMax) { - fractionStr += "0"; - } - } else if (fractionStr.length() > decimalMax) { - // Truncate if too many decimal places (alternative could be rounding) - fractionStr = fractionStr.substring(0, decimalMax); - } - fraction = Integer.parseInt(fractionStr); - } - - scaledValues[i] = whole * scale + fraction; - } - -// // 2. Process first element -// int first = scaledValues[0]; -// result[0] = first; -// -// // 3. Process subsequent elements with delta + ZigZag encoding -// int prev = first; -// for (int i = 1; i < size; i++) { -// int current = scaledValues[i]; -// int diff = current - prev; -// result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding -// prev = current; -// } - - return scaledValues; - } - - public static int[] sprintz(int[] numbers) { - int size = numbers.length; - int[] result = new int[size]; - - int first = numbers[0]; - result[0] = first; - - // 3. Process subsequent elements with delta + ZigZag encoding - int prev = first; - for (int i = 1; i < size; i++) { - int current = numbers[i]; - int diff = current - prev; - result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding - prev = current; - } - - return result; - } - - public static byte[] encodeBitPacking(int[] paddedArray, int[] bitWidths, int pack_size) { - List result = new ArrayList<>(); - - - - // 3. 对paddedArray进行bit-packing - int totalGroups = bitWidths.length; - - // 计算bit-packed数据的总字节数 - 修正计算方式 -// int totalBitPackedBytes = (cost_bits+7)/8; -// for (int i = 0; i < totalGroups; i++) { -// // 每组需要 ceil(8 * bitWidth / 8) = bitWidth 字节 -// totalBitPackedBytes += bitWidths[i]; -// } - - // 确保数组足够大，添加一些额外空间以防万一 - int max_bit_width = 0; - - for (int bitWidth : bitWidths) { - if (bitWidth > max_bit_width) { - max_bit_width = bitWidth; - } - } -// System.out.println(max_bit_width); -// if(max_bit_width ==0) { -// System.out.println(Arrays.toString(bitWidths)); -// } -// System.out.println(pack_size); -// System.out.println(totalGroups); - int totalBitPackedBytes = (max_bit_width*pack_size*totalGroups+7)/8; - byte[] bitPackedData = new byte[totalBitPackedBytes + totalGroups+32]; -// bitPackedData[0] = (byte) max_bit_width; - int encodePos = 0; - // 对每组数据进行bit-packing - for (int group = 0; group < totalGroups; group++) { - int startIndex = group * pack_size; - ArrayList groupData = new ArrayList<>(); - for (int i = 0; i < pack_size; i++) { - if (startIndex + i < paddedArray.length) { - groupData.add(paddedArray[startIndex + i]); - } else { - groupData.add(0); // 用0填充不足的部分 - } - } - - bitPackedData[encodePos++] = (byte) bitWidths[group]; - encodePos = bitPacking(groupData, 0,bitWidths[group] , encodePos, bitPackedData); - } - - - // 4. 将bit-packed数据写入结果（只写入实际使用的部分） - for (int i = 0; i < encodePos; i++) { - result.add(bitPackedData[i]); - } - - // 转换为byte数组返回 - byte[] finalResult = new byte[result.size()]; - for (int i = 0; i < result.size(); i++) { - finalResult[i] = result.get(i); - } - - return finalResult; - } - public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { - int blocksize= bitWidths.length; -// int minCost = Integer.MAX_VALUE; - - // Try all possible pack sizes from 1 to CHUNK_SIZE -// for (int p = 1; p <= blocksize; p++) { - int totalCost = 0; - int numPacks = (int) Math.ceil((double) blocksize / fixed_pack); - - // Calculate cost for each pack - for (int pack = 0; pack < numPacks; pack++) { - int start = pack * fixed_pack; - int end = Math.min(start + fixed_pack, blocksize); - - - // Find max bitWidth in current pack - int maxBitWidth = 0; - for (int i = start; i < end; i++) { - if (bitWidths[i] > maxBitWidth) { - maxBitWidth = bitWidths[i]; - } - } - - // Add to cost: 8 * p * maxBitWidth - totalCost += pack_size * (end-start) * maxBitWidth; - } - - // Add the chunk cost: 5 * CHUNK_SIZE / p - totalCost += 5 * blocksize / fixed_pack; - -// // Update minimum cost -// if (totalCost < minCost) { -// minCost = totalCost; -// } -// } - return totalCost; - } - - /** - * ZigZag解码 - */ - private static int zigzagDecode(int n) { - return (n >>> 1) ^ -(n & 1); - } - - /** - * 解压函数 - 从压缩数据中恢复原始整数数组 - */ -// public static int[] decodeBitPackingFull(byte[] compressedData, int originalLength, int packSize) { -// List decodedValues = new ArrayList<>(); -// int decodePos = 0; -// -// // 计算分组数量 -// int totalGroups = (originalLength + packSize - 1) / packSize; -// -// for (int group = 0; group < totalGroups; group++) { -// // 读取该组的位宽 -// int bitWidth = compressedData[decodePos++] & 0xFF; -// -// if (bitWidth == 0) { -// // 如果位宽为0，说明这组都是0 -// for (int i = 0; i < packSize; i++) { -// decodedValues.add(0); -// } -// continue; -// } -// -// // 解压该组数据 -// ArrayList groupData = new ArrayList<>(); -// unpack8Values(compressedData, decodePos, bitWidth, groupData); -// decodePos += bitWidth; -// -// // 添加到结果列表 -// for (int value : groupData) { -// decodedValues.add(value); -// } -// } -// -// // 转换为数组并截取原始长度（因为可能有填充） -// int[] result = new int[originalLength]; -// for (int i = 0; i < originalLength; i++) { -// result[i] = decodedValues.get(i); -// } -// -// return result; -// } - public static int[] decodeBitPackingFull(byte[] compressedData, int originalLength, int packSize) { - // 预分配结果数组，避免ArrayList的动态扩容开销 - int[] result = new int[originalLength]; - int resultIndex = 0; - int decodePos = 0; - - // 计算分组数量 - int totalGroups = (originalLength + packSize - 1) / packSize; - - // 预分配组数据数组，避免在循环中重复创建 - int[] groupData = new int[packSize]; - - for (int group = 0; group < totalGroups && resultIndex < originalLength; group++) { - // 读取该组的位宽 - int bitWidth = compressedData[decodePos++] & 0xFF; - - if (bitWidth == 0) { - // 如果位宽为0，说明这组都是0 - 直接填充0 - int fillCount = Math.min(packSize, originalLength - resultIndex); - // Arrays.fill比循环更快 - if (fillCount > 0) { - Arrays.fill(result, resultIndex, resultIndex + fillCount, 0); - resultIndex += fillCount; - } - continue; - } - - // 直接解压到预分配的groupData数组，避免ArrayList的开销 - int actualUnpacked = unpack8ValuesToArray(compressedData, decodePos, bitWidth, groupData); - decodePos += bitWidth; - - // 直接复制到结果数组，避免额外的循环 - int copyCount = Math.min(actualUnpacked, originalLength - resultIndex); - System.arraycopy(groupData, 0, result, resultIndex, copyCount); - resultIndex += copyCount; - } - - return result; - } - private static int unpack8ValuesToArray(byte[] encoded, int offset, int width, int[] result) { - int byteIdx = offset; - long buffer = 0; - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8 && valueIdx < result.length) { - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - while (totalBits >= width && valueIdx < 8 && valueIdx < result.length) { - result[valueIdx] = (int) (buffer >>> (totalBits - width)); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - - return valueIdx; // 返回实际解压的数量 - } - /** - * Sprintz解码 - 从差分编码恢复原始数据 - */ - public static int[] sprintzDecode(int[] encodedData) { - int size = encodedData.length; - int[] result = new int[size]; - - if (size == 0) return result; - - // 第一个元素是原始值 - result[0] = encodedData[0]; - - // 后续元素需要ZigZag解码和累加 - int prev = result[0]; - for (int i = 1; i < size; i++) { - int zigzagEncoded = encodedData[i]; - int diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 - result[i] = prev + diff; - prev = result[i]; - } - - return result; - } - public static double[] unscaleNumbers(int[] scaledValues, int decimalMax) { - double scale = Math.pow(10, decimalMax); - int size = scaledValues.length; - double[] result = new double[size]; - - for (int i = 0; i < size; i++) { - result[i] = scaledValues[i] / scale; - } - - return result; - } - public static int[] decompress(byte[] compressedData, int originalLength, int decimalMax, int packSize) { - // 1. 位解压 - int[] bitUnpacked = decodeBitPackingFull(compressedData, originalLength, packSize); - - // 2. Sprintz解码 - int[] sprintzDecoded = sprintzDecode(bitUnpacked); - - // 3. 缩放逆变换 -// double[] unscaled = unscaleNumbers(sprintzDecoded, decimalMax); - - return sprintzDecoded; - } - public static int[] decodeBitPacking(byte[] compressedData, int[] bitWidths, int pack_size, int originalLength) { - int[] result = new int[originalLength]; // 直接使用数组 - int resultIndex = 0; - int decodePos = 0; - - for (int group = 0; group < bitWidths.length && resultIndex < originalLength; group++) { - int bitWidth = compressedData[decodePos++] & 0xFF; - - // 预分配固定大小的列表 - ArrayList groupData = new ArrayList<>(pack_size); - unpack8Values(compressedData, decodePos, bitWidth, groupData); - - // 批量拷贝 - int copyLength = Math.min(pack_size, originalLength - resultIndex); - for (int i = 0; i < copyLength; i++) { - result[resultIndex++] = groupData.get(i); - } - - decodePos += bitWidth; - } - - return result; - } - - @Test - public void printDataTest() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); -// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/bitwidth"; - File outputDir = new File(outputDirstr); - -// RLDecisionModel trainedModel = trainModel(20, csvFilePath); - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; -// if(!file.getName().equals("Stocks-DE.csv")) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "BP", - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - List bitWidthRecords = new ArrayList<>(); - List sprintzBitWidthRecords = new ArrayList<>(); - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - bitWidths[scaledInts_i / 8] = bitWidth; - String[] record = { - String.valueOf(bitWidth), - }; - writer.writeRecord(record); - } - - } - - - writer.close(); -// break; - } - } - - @Test - public void printSprintzDataTest() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); -// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/bitwidth_sprintz"; - File outputDir = new File(outputDirstr); - -// RLDecisionModel trainedModel = trainModel(20, csvFilePath); - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; -// if(!file.getName().equals("Stocks-DE.csv")) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Sprintz", - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - List bitWidthRecords = new ArrayList<>(); - List sprintzBitWidthRecords = new ArrayList<>(); - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scalingInt); - - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(maxInGroup); - - bitWidths[scaledInts_i / 8] = bitWidth; - String[] record = { - String.valueOf(bitWidth), - }; - writer.writeRecord(record); - } - - } - - - writer.close(); -// break; - } - } - - public static void main(String[] args) throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); -// String csvFilePath = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/processed_data.csv"; - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz"; - File outputDir = new File(outputDirstr); - -// RLDecisionModel trainedModel = trainModel(20, csvFilePath); - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; -// if(!file.getName().equals("Stocks-DE.csv")) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; -// System.out.println(numbers.size()); - - - // 方法：强化学习 -// long modelStart = System.nanoTime(); - int modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; // 新增解码时间统计 - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scalingInt = scaleNumbers(chunkNumbers, decimalMax); - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scalingInt); - - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - bitWidths[scaledInts_i / 8] = bitWidth; - } -// System.out.println(Arrays.toString(bitWidths)); - int fixed_pack = CHUNK_SIZE / 40; -// int cur_cost = computeMinPackingCost(bitWidths,fixed_pack, 8); - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, 8); - int cur_cost = compressedData.length * 8; // 转换为bit数 -// System.out.println(cur_cost); -// PackingResult result = packOctads(bitWidths, model, null); // 禁用决策跟踪 - long duration = System.nanoTime() - startTime; - - long decodeStartTime = System.nanoTime(); - // 执行解压 - int[] decodedData = decodeBitPacking(compressedData, bitWidths, 8, scaledInts.length); - sprintzDecode(decodedData); -// int[] decompressed = decompress(compressedData, chunkNumbers.size(), decimalMax, 8); - - long decodeDuration = System.nanoTime() - decodeStartTime; - modelDecodeTime += decodeDuration; - - modelTime += (duration); - modelCost += cur_cost; -// if(i==0) -// for (int episode = 0; episode < 10; episode++) { -// trainEpisode(scaledInts, episode); -// } -// List optimalK = predictOptimalK(scaledInts); -// System.out.println("Optimal k sequence: " + optimalK); - } - - } - modelCost /=time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - modelDecodeTime /= time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); - double modelDecodeTime_throughput = (double)(numbers.size()*8000)/ (double) (modelDecodeTime); - String[] record = { - file.toString(), - "Sprintz", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); -// break; - } - } - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_SPRINTZ_vary_pack_size"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - // 方法：强化学习 - int modelCost = 0; - long modelTime = 0; - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - - List chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); - int cur_cost = compressedData.length * 8; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - String[] record = { - file.toString(), - "SPRINTZ", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - writer.close(); - } - } - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 -// int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); -// int[] scaledInts_all = scaleNumbers(numbers, decimalMax); - - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - -// 分批处理，每1024个元素一批 - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - int[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - - int currentIndex = 0; - for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); -// System.out.println(numbers.subList(0,1000)); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); - -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); - - int end = Math.min(i + chunkSize, numbers.size()); - int[] scaledInt = new int[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为pack_size的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; -// if(i==0){ -// System.out.println(Arrays.toString(paddedArray)); -// } - - - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; -// System.out.println(bitWidth); - } - - byte[] compressedData = encodeBitPacking(paddedArray, bitWidths, pack_size); - int cur_cost = compressedData.length * 8; - long duration = System.nanoTime() - startTime; - - modelTime += (duration); - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "BP", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); -// break; - } - } - -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java deleted file mode 100644 index ed1d315c7..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthSprintzTest.java +++ /dev/null @@ -1,916 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -public class RLEPackBitWidthSprintzTest { -// private static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data","test.csv","POI-lat.csv", -// "POI-lon.csv","Air-sensor.csv","Basel-temp.csv"); -static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - private static final int CHUNK_SIZE = 1024; - - public static int getCount(long long1, int mask) { - return ((int) (long1 & mask)); - } - - public static int bytes2Integer(byte[] encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded[i + start] & 0xFF; - value |= b; - } - return value; - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - // total bits which have read from 'buf' to 'buffer'. i.e., - // number of available bits to be decoded. - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - // If current available bits are not enough to decode one Integer, - // then add next byte from buf to 'buffer' until totalBits >= width - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - // If current available bits are enough to decode one Integer, - // then decode one Integer one by one until left bits in 'buffer' is - // not enough to decode one Integer. - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - - return encode_pos; - - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - public static int[] scaleNumbers(List numbers, int decimalMax) { - int scale = (int) Math.pow(10, decimalMax); - int size = numbers.size(); - int[] result = new int[size]; - - if (size == 0) { - return result; - } - - // 1. Parse all numbers and scale them up - int[] scaledValues = new int[size]; - for (int i = 0; i < size; i++) { - String numStr = numbers.get(i); - // Parse the number (handling both "123.456" and "123" cases) - String[] parts = numStr.split("\\."); - int whole = Integer.parseInt(parts[0]); - - // Handle fractional part - int fraction = 0; - if (parts.length > 1) { - String fractionStr = parts[1]; - // Pad with zeros if necessary to ensure proper scaling - if (fractionStr.length() < decimalMax) { - while (fractionStr.length() < decimalMax) { - fractionStr += "0"; - } - } else if (fractionStr.length() > decimalMax) { - // Truncate if too many decimal places (alternative could be rounding) - fractionStr = fractionStr.substring(0, decimalMax); - } - fraction = Integer.parseInt(fractionStr); - } - - scaledValues[i] = whole * scale + fraction; - } - return scaledValues; - } -// private static int[] scaleNumbers(List numbers, int decimalMax) { -// // 1. 预先计算缩放因子 -// BigDecimal scale = BigDecimal.TEN.pow(decimalMax); -// int size = numbers.size(); -// int[] result = new int[size]; -// -// if (size == 0) { -// return result; -// } -// -// // 2. 单次遍历完成所有转换和最小值查找 -// BigDecimal min = null; -// BigDecimal[] scaledValues = new BigDecimal[size]; -// -// for (int i = 0; i < size; i++) { -// BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); -// scaledValues[i] = val; -// if (min == null || val.compareTo(min) < 0) { -// min = val; -// } -// } -// -// // 3. 处理第一个元素 -// BigDecimal first = scaledValues[0].subtract(min); -// result[0] = first.toBigInteger().intValue(); -// -// // 4. 处理后续元素（差分+ZigZag） -// for (int i = 1; i < size; i++) { -// BigDecimal current = scaledValues[i].subtract(min); -// result[i]=current.toBigInteger().intValue(); -// } -// -// return result; -// } - public static int[] sprintz(int[] numbers) { - int size = numbers.length; - int[] result = new int[size]; - - int first = numbers[0]; - result[0] = first; - - // 3. Process subsequent elements with delta + ZigZag encoding - int prev = first; - for (int i = 1; i < size; i++) { - int current = numbers[i]; - int diff = current - prev; - result[i] = (diff << 1) ^ (diff >> 31); // ZigZag encoding - prev = current; - } - - return result; - } - - - /** - * Sprintz解码 - 从差分编码恢复原始数据 - */ - public static int[] sprintzDecode(int[] encodedData) { - int size = encodedData.length; - int[] result = new int[size]; - - if (size == 0) return result; - - // 第一个元素是原始值 - result[0] = encodedData[0]; - - // 后续元素需要ZigZag解码和累加 - int prev = result[0]; - for (int i = 1; i < size; i++) { - int zigzagEncoded = encodedData[i]; - int diff = (zigzagEncoded >>> 1) ^ -(zigzagEncoded & 1); // ZigZag解码 - result[i] = prev + diff; - prev = result[i]; - } - - return result; - } - - public static int[] decodeRLE(byte[] data, int startPos, int runCount) { - List bitWidths = new ArrayList<>(); - - for (int i = 0; i < runCount; i++) { - int runLength = data[startPos + i * 2] & 0xFF; - int value = data[startPos + i * 2 + 1] & 0xFF; - - // 重复添加runLength次value - for (int j = 0; j < runLength; j++) { - bitWidths.add(value); - } - } - - // 转换为数组 - int[] result = new int[bitWidths.size()]; - for (int i = 0; i < bitWidths.size(); i++) { - result[i] = bitWidths.get(i); - } - return result; - } - // 新增的解压函数 - public static int[] decodeBitPackingWithRLE(byte[] compressedData, int originalLength, int pack_size) { - List result = new ArrayList<>(); - int pos = 0; - - // 1. 解析RLE编码的bitWidths - // 读取run_count（4字节） - int runCount = bytes2Integer(compressedData, pos, 4); - pos += 4; - - // 解析RLE游程 - int[] bitWidths = decodeRLE(compressedData, pos, runCount); - pos += runCount * 2; // 每个游程占2字节 - - // 2. 解压bit-packed数据 - int totalGroups = bitWidths.length; - - for (int group = 0; group < totalGroups; group++) { - int bitWidth = bitWidths[group]; - - // 解压当前分组 - ArrayList groupData = new ArrayList<>(); - unpack8Values(compressedData, pos, bitWidth, groupData); - - // 添加解压出的数据 - for (int i = 0; i < pack_size; i++) { - if (result.size() < originalLength) { - result.add(groupData.get(i)); - } - } - - pos += bitWidth; - } - - // 转换为数组返回 - int[] decodedArray = new int[result.size()]; - for (int i = 0; i < result.size(); i++) { - decodedArray[i] = result.get(i); - } - return decodedArray; - } - - /** - * 实际的压缩编码函数：将paddedArray按照bitWidths进行bit-packing，并对bitWidths进行RLE编码 - */ - public static byte[] encodeBitPackingWithRLE(int[] paddedArray, int[] bitWidths, int pack_size, int cost_bits) { - List result = new ArrayList<>(); - - // 1. 对bitWidths进行RLE编码 - List rleEncoded = encodeRLE(bitWidths); - - // 2. 将RLE编码的bitWidths写入结果 - // 首先写入RLE数据的长度（4字节） -// int rleLength = rleEncoded.size(); -// result.add((byte) (rleLength >> 24)); -// result.add((byte) (rleLength >> 16)); -// result.add((byte) (rleLength >> 8)); -// result.add((byte) rleLength); - - // 写入RLE数据 - result.addAll(rleEncoded); - - // 3. 对paddedArray进行bit-packing - int totalGroups = bitWidths.length; - - // 计算bit-packed数据的总字节数 - 修正计算方式 - int totalBitPackedBytes = (cost_bits+7)/8; -// for (int i = 0; i < totalGroups; i++) { -// // 每组需要 ceil(8 * bitWidth / 8) = bitWidth 字节 -// totalBitPackedBytes += bitWidths[i]; -// } - - // 确保数组足够大，添加一些额外空间以防万一 - byte[] bitPackedData = new byte[totalBitPackedBytes + 32]; - int encodePos = 0; -// System.out.println(Arrays.toString(bitWidths)); -// System.out.println(result.size()); - // 对每组数据进行bit-packing - for (int group = 0; group < totalGroups; group++) { - int startIndex = group * pack_size; - ArrayList groupData = new ArrayList<>(); - for (int i = 0; i < pack_size; i++) { - if (startIndex + i < paddedArray.length) { - groupData.add(paddedArray[startIndex + i]); - } else { - groupData.add(0); // 用0填充不足的部分 - } - } - - // 确保不会越界 -// if (encodePos + bitWidths[group] <= bitPackedData.length) { - encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); -// } else { -// // 如果空间不足，扩展数组 -// byte[] newBitPackedData = new byte[bitPackedData.length + 32]; -// System.arraycopy(bitPackedData, 0, newBitPackedData, 0, bitPackedData.length); -// bitPackedData = newBitPackedData; -// encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); -// } - } - - // 4. 将bit-packed数据写入结果（只写入实际使用的部分） - for (int i = 0; i < encodePos; i++) { - result.add(bitPackedData[i]); - } - - // 转换为byte数组返回 - byte[] finalResult = new byte[result.size()]; - for (int i = 0; i < result.size(); i++) { - finalResult[i] = result.get(i); - } - - return finalResult; - } - - /** - * RLE编码bitWidths数组 - * chunksize = 1024 - * packsize = 8 - * runlength = 128 - * runcount = - */ - public static List encodeRLE(int[] bitWidths) { - List result = new ArrayList<>(); - - if (bitWidths.length == 0) { - return result; - } - int length_bitWidths_list = bitWidths.length; -// int currentValue = bitWidths[0]; -// int runLength = 1; - int run_count = 0; - - int[] run_lengths = new int[length_bitWidths_list]; - int[] run_values = new int[length_bitWidths_list]; - int pre_bit_width = bitWidths[0]; - int pre_run_length = 1; - - for (int i = 1; i < length_bitWidths_list; i++) { - if (bitWidths[i] == pre_bit_width) { - pre_run_length++; - } else { - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - // 写入当前游程 -// encodeRLERun(result, runLength, currentValue); - pre_bit_width = bitWidths[i]; - pre_run_length = 1; - } - } - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - - result.add((byte) (run_count >> 24)); - result.add((byte) (run_count >> 16)); - result.add((byte) (run_count >> 8)); - result.add((byte) run_count); - for (int i = 0; i < run_count; i++) { - encodeRLERun(result, run_lengths[i], run_values[i]); - } - // 写入最后一个游程 -// encodeRLERun(result, runLength, currentValue); - - return result; - } - - /** - * 编码单个RLE游程 - */ - private static void encodeRLERun(List result, int runLength, int value) { - // 使用变长编码存储游程长度 -// while (runLength > 0) { -// int byteValue = runLength & 0xFF; // 取7位 -// runLength >>= 7; -// if (runLength > 0) { -// byteValue |= 0x80; // 设置最高位表示还有后续字节 -// } -// result.add((byte) (runLength >> 24)); -// result.add((byte) (runLength >> 16)); -// result.add((byte) (runLength >> 8)); - result.add((byte) runLength); -// result.add((byte) (value >> 24)); -// result.add((byte) (value >> 16)); -// result.add((byte) (value >> 8)); - result.add((byte) value); - - } - - public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { - int blocksize= bitWidths.length; - - int totalCost = 0; - int numBlocks = (int) Math.ceil((double) blocksize / fixed_pack); - // RLE compress bit width series: rle_count 8 bits, (run length 8 bits, bit width value: 6 bits) * run_count - - // Calculate cost for each pack - for (int pack = 0; pack < numBlocks; pack++) { - int start = pack * fixed_pack; - int end = Math.min(start + fixed_pack, blocksize); - int cur_block_size = end - start; - - // Find max bitWidth in current pack - int maxBitWidth = 0; - int[] run_lengths = new int[cur_block_size]; - int[] run_values = new int[cur_block_size]; - int run_count = 0; - int pre_bit_width = bitWidths[start]; - int pre_run_length = 1; - totalCost += pack_size * bitWidths[start]; - - for (int i = start+1; i < end; i++) { - totalCost += pack_size * bitWidths[i]; - if(pre_bit_width == bitWidths[i]) { - pre_run_length++; - } else { - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - pre_bit_width = bitWidths[i]; - pre_run_length = 1; - } - } - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - -// System.out.println(run_count); - totalCost += 64; - for (int i = 0; i < run_count; i++) { - totalCost += 32; - } - - - } - System.out.println(blocksize); - // Store max bit width -// totalCost += 5 * blocksize / fixed_pack; - - - return totalCost; - } - - public static void main(String[] args) throws IOException { - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_sprintz"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - int modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / 8] = bitWidth; - cost_bits += (bitWidth*8); - } - - - // 替换computeMinPackingCost为实际的压缩编码 - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, 8,cost_bits); - int cur_cost = compressedData.length * 8; // 转换为bit数 -// System.out.println(cur_cost); - long duration = System.nanoTime() - startTime; - - long decodeStartTime = System.nanoTime(); - int[] decompressedPacked = decodeBitPackingWithRLE(compressedData, scaledInts.length, 8); - int[] decompressedScaled = sprintzDecode(decompressedPacked); - long decodeDuration = System.nanoTime() - decodeStartTime; - - modelTime += duration; - modelDecodeTime += decodeDuration; - modelCost += cur_cost; - } - } - modelCost /= time_of_repeat; - modelTime = modelTime / time_of_repeat; - modelDecodeTime = modelDecodeTime / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000) / (double) (modelTime); - double decodeThroughput = (double) (numbers.size() * 8000) / (double) modelDecodeTime; - String[] record = { - file.toString(), - "BP+RLE-Sprintz", - String.valueOf(modelTime_throughput), - String.valueOf(decodeThroughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); -// break; - } - } - - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_SPRINTZ_RLE_vary_pack_size"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++){ - int pack_size = (int) Math.pow(2,pack_size_exp); - int modelCost = 0; - long modelTime = 0; - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInt = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - cost_bits += (bitWidth*pack_size); - } - - int fixed_block = CHUNK_SIZE / pack_size; - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); - int cur_cost = compressedData.length * 8; // 转换为bit数 - - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /=time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); - String[] record = { - file.toString(), - "SPRINTZ+RLE", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - - writer.close(); - } - - } - // 新增方法：测试不同chunk size的表现 - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_sprintz_RLE_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - int[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - - int currentIndex = 0; - for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); -// -// int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - int end = Math.min(i + chunkSize, numbers.size()); - int[] scaledInt = new int[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; - cost_bits += (bitWidth * pack_size); - } - - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); - int cur_cost = compressedData.length * 8; - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = modelTime / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "Sprintz+RLE", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); - } - } - -} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java deleted file mode 100644 index 60bf431ac..000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/RLEPackBitWidthTest.java +++ /dev/null @@ -1,858 +0,0 @@ -package org.apache.iotdb.tsfile.encoding; - -import com.csvreader.CsvReader; -import com.csvreader.CsvWriter; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class RLEPackBitWidthTest { - static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv","POI-lat.csv", - "POI-lon.csv","Basel-wind.csv","Basel-temp.csv","Air-sensor.csv"); - private static final int CHUNK_SIZE = 1024; - - public static int getBitWith(int num) { - if (num == 0) - return 1; - else - return 32 - Integer.numberOfLeadingZeros(num); - } - - public static int getCount(long long1, int mask) { - return ((int) (long1 & mask)); - } - - public static int getUniqueValue(long long1, int left_shift) { - return ((int) ((long1) >> left_shift)); - } - - public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer); - } - - private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) { - cur_byte[encode_pos] = (byte) (integer >> 24); - cur_byte[encode_pos + 1] = (byte) (integer >> 16); - cur_byte[encode_pos + 2] = (byte) (integer >> 8); - cur_byte[encode_pos + 3] = (byte) (integer); - } - - public static int bytes2Integer(byte[] encoded, int start, int num) { - int value = 0; - if (num > 4) { - System.out.println("bytes2Integer error"); - return 0; - } - for (int i = 0; i < num; i++) { - value <<= 8; - int b = encoded[i + start] & 0xFF; - value |= b; - } - return value; - } - - private static long bytesLong2Integer(byte[] encoded, int decode_pos) { - long value = 0; - for (int i = 0; i < 4; i++) { - value <<= 8; - int b = encoded[i + decode_pos] & 0xFF; - value |= b; - } - return value; - } - - public static void pack8Values(ArrayList values, int offset, int width, int encode_pos, - byte[] encoded_result) { - int bufIdx = 0; - int valueIdx = offset; - // remaining bits for the current unfinished Integer - int leftBit = 0; - - while (valueIdx < 8 + offset) { - // buffer is used for saving 32 bits as a part of result - int buffer = 0; - // remaining size of bits in the 'buffer' - int leftSize = 32; - - // encode the left bits of current Integer to 'buffer' - if (leftBit > 0) { - buffer |= (values.get(valueIdx) << (32 - leftBit)); - leftSize -= leftBit; - leftBit = 0; - valueIdx++; - } - - while (leftSize >= width && valueIdx < 8 + offset) { - // encode one Integer to the 'buffer' - buffer |= (values.get(valueIdx) << (leftSize - width)); - leftSize -= width; - valueIdx++; - } - // If the remaining space of the buffer can not save the bits for one Integer, - if (leftSize > 0 && valueIdx < 8 + offset) { - // put the first 'leftSize' bits of the Integer into remaining space of the - // buffer - buffer |= (values.get(valueIdx) >>> (width - leftSize)); - leftBit = width - leftSize; - } - - // put the buffer into the final result - for (int j = 0; j < 4; j++) { - encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); - encode_pos++; - bufIdx++; - if (bufIdx >= width) { - return; - } - } - } - - } - - public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList result_list) { - int byteIdx = offset; - long buffer = 0; - // total bits which have read from 'buf' to 'buffer'. i.e., - // number of available bits to be decoded. - int totalBits = 0; - int valueIdx = 0; - - while (valueIdx < 8) { - // If current available bits are not enough to decode one Integer, - // then add next byte from buf to 'buffer' until totalBits >= width - while (totalBits < width) { - buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); - byteIdx++; - totalBits += 8; - } - - // If current available bits are enough to decode one Integer, - // then decode one Integer one by one until left bits in 'buffer' is - // not enough to decode one Integer. - while (totalBits >= width && valueIdx < 8) { - result_list.add((int) (buffer >>> (totalBits - width))); - valueIdx++; - totalBits -= width; - buffer = buffer & ((1L << totalBits) - 1); - } - } - } - - public static int bitPacking(ArrayList numbers, int start, int bit_width, int encode_pos, - byte[] encoded_result) { - int block_num = (numbers.size() - start) / 8; - for (int i = 0; i < block_num; i++) { - pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result); - encode_pos += bit_width; - } - - return encode_pos; - - } - - public static ArrayList decodeBitPacking( - byte[] encoded, int decode_pos, int bit_width, int block_size) { - ArrayList result_list = new ArrayList<>(); - int block_num = (block_size - 1) / 8; - - for (int i = 0; i < block_num; i++) { // bitpacking - unpack8Values(encoded, decode_pos, bit_width, result_list); - decode_pos += bit_width; - } - return result_list; - } - - // 新增的解压函数 - public static int[] decodeBitPackingWithRLE(byte[] compressedData, int originalLength, int pack_size) { - List result = new ArrayList<>(); - int pos = 0; - - // 1. 解析RLE编码的bitWidths - // 读取run_count（4字节） - int runCount = bytes2Integer(compressedData, pos, 4); - pos += 4; - - // 解析RLE游程 - int[] bitWidths = decodeRLE(compressedData, pos, runCount); - pos += runCount * 2; // 每个游程占2字节 - - // 2. 解压bit-packed数据 - int totalGroups = bitWidths.length; - - for (int group = 0; group < totalGroups; group++) { - int bitWidth = bitWidths[group]; - - // 解压当前分组 - ArrayList groupData = new ArrayList<>(); - unpack8Values(compressedData, pos, bitWidth, groupData); - - // 添加解压出的数据 - for (int i = 0; i < pack_size; i++) { - if (result.size() < originalLength) { - result.add(groupData.get(i)); - } - } - - pos += bitWidth; - } - - // 转换为数组返回 - int[] decodedArray = new int[result.size()]; - for (int i = 0; i < result.size(); i++) { - decodedArray[i] = result.get(i); - } - return decodedArray; - } - - // 新增的RLE解码函数 - public static int[] decodeRLE(byte[] data, int startPos, int runCount) { - List bitWidths = new ArrayList<>(); - - for (int i = 0; i < runCount; i++) { - int runLength = data[startPos + i * 2] & 0xFF; - int value = data[startPos + i * 2 + 1] & 0xFF; - - // 重复添加runLength次value - for (int j = 0; j < runLength; j++) { - bitWidths.add(value); - } - } - - // 转换为数组 - int[] result = new int[bitWidths.size()]; - for (int i = 0; i < bitWidths.size(); i++) { - result[i] = bitWidths.get(i); - } - return result; - } - - private static int[] scaleNumbers(List numbers, int decimalMax) { - // 1. 预先计算缩放因子 - BigDecimal scale = BigDecimal.TEN.pow(decimalMax); - int size = numbers.size(); - int[] result = new int[size]; - - if (size == 0) { - return result; - } - - // 2. 单次遍历完成所有转换和最小值查找 - BigDecimal min = null; - BigDecimal[] scaledValues = new BigDecimal[size]; - - for (int i = 0; i < size; i++) { - BigDecimal val = new BigDecimal(numbers.get(i)).multiply(scale); - scaledValues[i] = val; - if (min == null || val.compareTo(min) < 0) { - min = val; - } - } - - // 3. 处理第一个元素 - BigDecimal first = scaledValues[0].subtract(min); - result[0] = first.toBigInteger().intValue(); - - // 4. 处理后续元素（差分+ZigZag） - for (int i = 1; i < size; i++) { - BigDecimal current = scaledValues[i].subtract(min); - result[i]=current.toBigInteger().intValue(); - } - - return result; - } - - /** - * 实际的压缩编码函数：将paddedArray按照bitWidths进行bit-packing，并对bitWidths进行RLE编码 - */ - public static byte[] encodeBitPackingWithRLE(int[] paddedArray, int[] bitWidths, int pack_size, int cost_bits) { - List result = new ArrayList<>(); - - // 1. 对bitWidths进行RLE编码 - List rleEncoded = encodeRLE(bitWidths); - - // 2. 将RLE编码的bitWidths写入结果 - // 写入RLE数据 - result.addAll(rleEncoded); - - // 3. 对paddedArray进行bit-packing - int totalGroups = bitWidths.length; - - // 计算bit-packed数据的总字节数 - 修正计算方式 - int totalBitPackedBytes = (cost_bits+7)/8; - - // 确保数组足够大，添加一些额外空间以防万一 - byte[] bitPackedData = new byte[totalBitPackedBytes + 32]; - int encodePos = 0; - - // 对每组数据进行bit-packing - for (int group = 0; group < totalGroups; group++) { - int startIndex = group * pack_size; - ArrayList groupData = new ArrayList<>(); - for (int i = 0; i < pack_size; i++) { - if (startIndex + i < paddedArray.length) { - groupData.add(paddedArray[startIndex + i]); - } else { - groupData.add(0); // 用0填充不足的部分 - } - } - - encodePos = bitPacking(groupData, 0, bitWidths[group], encodePos, bitPackedData); - } - - // 4. 将bit-packed数据写入结果（只写入实际使用的部分） - for (int i = 0; i < encodePos; i++) { - result.add(bitPackedData[i]); - } - - // 转换为byte数组返回 - byte[] finalResult = new byte[result.size()]; - for (int i = 0; i < result.size(); i++) { - finalResult[i] = result.get(i); - } - - return finalResult; - } - - /** - * RLE编码bitWidths数组 - * chunksize = 1024 - * packsize = 8 - * runlength = 128 - * runcount = - */ - public static List encodeRLE(int[] bitWidths) { - List result = new ArrayList<>(); - - if (bitWidths.length == 0) { - return result; - } - int length_bitWidths_list = bitWidths.length; - int run_count = 0; - - int[] run_lengths = new int[length_bitWidths_list]; - int[] run_values = new int[length_bitWidths_list]; - int pre_bit_width = bitWidths[0]; - int pre_run_length = 1; - - for (int i = 1; i < length_bitWidths_list; i++) { - if (bitWidths[i] == pre_bit_width) { - pre_run_length++; - } else { - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - pre_bit_width = bitWidths[i]; - pre_run_length = 1; - } - } - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - - result.add((byte) (run_count >> 24)); - result.add((byte) (run_count >> 16)); - result.add((byte) (run_count >> 8)); - result.add((byte) run_count); - for (int i = 0; i < run_count; i++) { - encodeRLERun(result, run_lengths[i], run_values[i]); - } - - return result; - } - - /** - * 编码单个RLE游程 - */ - private static void encodeRLERun(List result, int runLength, int value) { - result.add((byte) (runLength >> 24)); - result.add((byte) (runLength >> 16)); - result.add((byte) (runLength >> 8)); - result.add((byte) runLength); - result.add((byte) (runLength >> 24)); - result.add((byte) (value >> 16)); - result.add((byte) (value >> 8)); - result.add((byte) value); - } - - public static int computeMinPackingCost(int[] bitWidths, int fixed_pack, int pack_size) { - int blocksize= bitWidths.length; - - int totalCost = 0; - int numBlocks = (int) Math.ceil((double) blocksize / fixed_pack); - // RLE compress bit width series: rle_count 8 bits, (run length 8 bits, bit width value: 6 bits) * run_count - - // Calculate cost for each pack - for (int pack = 0; pack < numBlocks; pack++) { - int start = pack * fixed_pack; - int end = Math.min(start + fixed_pack, blocksize); - int cur_block_size = end - start; - - // Find max bitWidth in current pack - int maxBitWidth = 0; - int[] run_lengths = new int[cur_block_size]; - int[] run_values = new int[cur_block_size]; - int run_count = 0; - int pre_bit_width = bitWidths[start]; - int pre_run_length = 1; - totalCost += pack_size * bitWidths[start]; - - for (int i = start+1; i < end; i++) { - totalCost += pack_size * bitWidths[i]; - if(pre_bit_width == bitWidths[i]) { - pre_run_length++; - } else { - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - pre_bit_width = bitWidths[i]; - pre_run_length = 1; - } - } - run_lengths[run_count] = pre_run_length; - run_values[run_count++] = pre_bit_width; - - totalCost += 64; - for (int i = 0; i < run_count; i++) { - totalCost += 32; - } - } - System.out.println(blocksize); - - return totalCost; - } - - public static void main(String[] args) throws IOException { - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - // 更新表头，增加解压吞吐率列 - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; // 新增：解压时间统计 - - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % 8; - int paddingLength = (remainder == 0) ? 0 : 8 - remainder; - - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / 8]; - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += 8) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + 8; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / 8] = bitWidth; - cost_bits += (bitWidth*8); - } - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, 8,cost_bits); - long cur_cost = compressedData.length * 8; // 转换为bit数 - - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += cur_cost; - - // 新增：测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingWithRLE(compressedData, scaledInts.length, 8); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - -// // 可选：验证解压数据的正确性（只在第一次重复时验证） -// if (j == 0) { -// boolean correct = true; -// for (int k = 0; k < scaledInts.length; k++) { -// if (scaledInts[k] != decodedData[k]) { -// correct = false; -// System.err.println("Decompression error at index " + k + -// ": expected " + scaledInts[k] + ", got " + decodedData[k]); -// break; -// } -// } -//// if (correct) { -//// System.out.println("Decompression verified successfully for chunk " + (i/CHUNK_SIZE)); -//// } -// } - } - } - modelCost /= time_of_repeat; - modelTime = modelTime / time_of_repeat; - modelDecodeTime = modelDecodeTime / time_of_repeat; // 平均解压时间 - - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000L) / (double) (modelTime); // points per second - double modelDecodeTime_throughput = (double)(numbers.size()*8000L) / (double) (modelDecodeTime); // points per second - - // 更新输出记录，包含解压吞吐率 - String[] record = { - file.toString(), - "BP+RLE", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); - -// System.out.println("Encoding throughput: " + modelTime_throughput + " points/s"); -// System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " points/s"); -// System.out.println("Compression ratio: " + model_ratio); - } - } - - @Test - public void TestVarPackSize() throws IOException { - // 示例数据（实际应替换为真实时间序列） - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_vary_pack_size"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); // write header to output file - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = 50; - - for(int pack_size_exp = 3; pack_size_exp < 10; pack_size_exp++){ - int pack_size = (int) Math.pow(2,pack_size_exp); - System.out.println(pack_size); - int modelCost = 0; - long modelTime = 0; - for(int j=0;j chunkNumbers = numbers.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())); - if(chunkNumbers.size()==1 || chunkNumbers.size()==2) - continue; - - int decimalMax = decimalPlaces.subList(i, Math.min(i + CHUNK_SIZE, numbers.size())) - .stream().max(Integer::compare).orElse(0); - - int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - // 创建新数组，长度补齐为8的倍数 - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; // 存储每8个值的位宽结果 - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - // 1. 找出当前8个元素中的最大值 - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - - // 2. 计算该最大值的去头零位宽 - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - - // 3. 存储结果 - bitWidths[scaledInts_i / pack_size] = bitWidth; - cost_bits += (bitWidth*pack_size); - } - - int fixed_block = CHUNK_SIZE / pack_size; - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); - int cur_cost = compressedData.length * 8; // 转换为bit数 - - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - } - - } - modelCost /=time_of_repeat; - modelTime = (modelTime)/time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size()*64); - double modelTime_throughput = (double)(numbers.size()*8000)/ (double) (modelTime); - String[] record = { - file.toString(), - "BP+RLE", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - - writer.close(); - } - - } - - // 新增方法：测试不同chunk size的表现 - @Test - public void TestVariableChunkSize() throws IOException { - System.out.println("\nPerformance Testing with Variable Chunk Sizes..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-block/elf_resources/output_BPRLE_vary_m"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - - // 定义要测试的chunk sizes (m*8 where m is 16, 32, 64, 128, 256, 512, 1024) - int[] chunkSizes = {16*8, 32*8, 64*8, 128*8, 256*8, 512*8, 1024*8}; - - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println("Processing " + file.getName() + " with variable chunk sizes..."); - String Output = outputDirstr+"/"+file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "m", - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Points", - "Compressed Size", - "Pack Size", - "Compression Ratio" - }; - writer.writeRecord(head); - - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - } - decimalPlaces.add(decimal); - } - } - } - - int time_of_repeat = 50; // 减少重复次数以加快测试速度 - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - // 分批处理，每1024个元素一批 - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - int[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - - int currentIndex = 0; - for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - - // 测试每个chunk size - for (int chunkSize : chunkSizes) { - System.out.println("Testing chunk size: " + chunkSize); - - for(int pack_size_exp = 3; pack_size_exp < 4; pack_size_exp++) { - int pack_size = (int) Math.pow(2, pack_size_exp); - int modelCost = 0; - long modelTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += chunkSize) { - -// List chunkNumbers = numbers.subList(i, Math.min(i + chunkSize, numbers.size())); -// if (chunkNumbers.size() == 1 || chunkNumbers.size() == 2) -// continue; -// -// int decimalMax = decimalPlaces.subList(i, Math.min(i + chunkSize, numbers.size())) -// .stream().max(Integer::compare).orElse(0); -// -// int[] scaledInts = scaleNumbers(chunkNumbers, decimalMax); - int end = Math.min(i + chunkSize, numbers.size()); - int[] scaledInts = new int[end-i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - int remainder = scaledInts.length % pack_size; - int paddingLength = (remainder == 0) ? 0 : pack_size - remainder; - - int[] paddedArray = new int[scaledInts.length + paddingLength]; - System.arraycopy(scaledInts, 0, paddedArray, 0, scaledInts.length); - int actual_length = paddedArray.length; - int[] bitWidths = new int[actual_length / pack_size]; - - int cost_bits = 0; - for (int scaledInts_i = 0; scaledInts_i < actual_length; scaledInts_i += pack_size) { - int maxInGroup = 0; - for (int scaledInts_j = scaledInts_i; scaledInts_j < scaledInts_i + pack_size; scaledInts_j++) { - if (paddedArray[scaledInts_j] > maxInGroup) { - maxInGroup = paddedArray[scaledInts_j]; - } - } - int bitWidth = 32 - Integer.numberOfLeadingZeros(maxInGroup); - bitWidths[scaledInts_i / pack_size] = bitWidth; - cost_bits += (bitWidth * pack_size); - } - - byte[] compressedData = encodeBitPackingWithRLE(paddedArray, bitWidths, pack_size, cost_bits); - int cur_cost = compressedData.length * 8; - long duration = System.nanoTime() - startTime; - modelTime += duration; - modelCost += cur_cost; - } - } - - modelCost /= time_of_repeat; - modelTime = modelTime / time_of_repeat; - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000) / (double) (modelTime); - - String[] record = { - String.valueOf(chunkSize/8), - file.toString(), - "BP+RLE", - String.valueOf(modelTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(pack_size), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - } - } - writer.close(); - } - } -} \ No newline at end of file From cd2cb458f093dc9b09926ce404e4fc2c0f85d83e Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:19:31 +0800 Subject: [PATCH 3/9] revision v1.0 --- .gitignore | 3 + java/tsfile/pom.xml | 6 + .../tsfile/common/conf/TSFileConfig.java | 22 + .../encoding/bitpacking/LongPacker.java | 81 + .../encoding/decoder/LongSprintzDecoder.java | 205 +- .../encoding/encoder/LongSprintzEncoder.java | 231 +- .../encoding/encoder/SprintzEncoder.java | 2 +- .../optimal/SprintzOptimalPackSize.java | 121 + .../encoding/TsFileEncodingException.java | 4 + .../tsfile/read/reader/page/PageReader.java | 2 + .../org/apache/tsfile/encoding/ALPTest.java | 1021 +++++++ .../encoding/AllNo8PacksizeOptimal.java | 2549 ++++++++++++++++- .../apache/tsfile/encoding/CuSZpCpuTest.java | 576 ++++ .../apache/tsfile/encoding/HBPIndexLong.java | 188 ++ .../tsfile/encoding/HBPIndexLongTest.java | 311 ++ .../SprintzOptimalPackSizeBenchmarkTest.java | 220 ++ .../apache/tsfile/write/CsvReadWriteTest.java | 770 +++++ optimal_pack_prune | Bin 0 -> 61040 bytes 18 files changed, 6172 insertions(+), 140 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/ALPTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/SprintzOptimalPackSizeBenchmarkTest.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/write/CsvReadWriteTest.java create mode 100755 optimal_pack_prune diff --git a/.gitignore b/.gitignore index 4fa45012a..8b1c86391 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ docs/src/.vuepress/.temp/ docs/src/.vuepress/dist/ docs/pnpm-lock.yaml +# python venv (for pyfastlanes / FastLanesTest) +.venv/ + # python files python/build python/dist diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml index 86f405bc7..6793a673c 100644 --- a/java/tsfile/pom.xml +++ b/java/tsfile/pom.xml @@ -69,6 +69,12 @@ junit test + + net.sourceforge.javacsv + javacsv + 2.0 + test + org.mockito mockito-core diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 24ab1428c..ad0c2bd09 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -159,6 +159,12 @@ public class TSFileConfig implements Serializable { /** Default block size of two-diff. delta encoding is 128. */ private int deltaBlockSize = 128; + /** Sprintz encoding block size for bit-packing (default 8). Use SprintzOptimalPackSize to find optimal value. */ + private int sprintzBlockSize = 8; + + /** When true, each block automatically finds optimal pack size for bit-packing (new algorithm). */ + private boolean sprintzUseOptimalPackSize = false; + /** Default frequency type is SINGLE_FREQ. */ private String freqType = "SINGLE_FREQ"; @@ -701,6 +707,22 @@ public String getSprintzPredictScheme() { return "fire"; } + public int getSprintzBlockSize() { + return sprintzBlockSize; + } + + public void setSprintzBlockSize(int sprintzBlockSize) { + this.sprintzBlockSize = Math.max(1, Math.min(32, sprintzBlockSize)); + } + + public boolean isSprintzUseOptimalPackSize() { + return sprintzUseOptimalPackSize; + } + + public void setSprintzUseOptimalPackSize(boolean sprintzUseOptimalPackSize) { + this.sprintzUseOptimalPackSize = sprintzUseOptimalPackSize; + } + public String getHdfsFile() { return hdfsFile; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/bitpacking/LongPacker.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/bitpacking/LongPacker.java index 9d3229773..5d52ca74c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/bitpacking/LongPacker.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/bitpacking/LongPacker.java @@ -101,6 +101,48 @@ public void pack8Values(long[] values, int offset, byte[] buf) { } } + /** + * Encode N Long values with specified bit-width to bytes. Supports variable pack sizes. + * + * @param values - array where values are in + * @param offset - offset of first value to be encoded + * @param n - number of values to encode (1-32) + * @param buf - encoded bytes, size must be ceil(n * width / 8) + */ + public void packNValues(long[] values, int offset, int n, byte[] buf) { + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + int byteLimit = (n * width + 7) / 8; + + while (valueIdx < n + offset && bufIdx < byteLimit) { + long buffer = 0; + int leftSize = 64; + + if (leftBit > 0) { + buffer |= (values[valueIdx] << (64 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < n + offset) { + buffer |= (values[valueIdx] << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + if (leftSize > 0 && valueIdx < n + offset) { + buffer |= (values[valueIdx] >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 8 && bufIdx < byteLimit; j++) { + buf[bufIdx] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF); + bufIdx++; + } + } + } + /** * decode values from byte array. * @@ -149,6 +191,45 @@ public void unpack8Values(byte[] buf, int offset, long[] values) { } } + /** + * Decode N long values from byte array. + * + * @param buf - array where bytes are in, size must be at least offset + ceil(n * width / 8) + * @param offset - offset of first byte to be decoded + * @param n - number of values to decode + * @param values - decoded result, size must be at least n + */ + public void unpackNValues(byte[] buf, int offset, int n, long[] values) { + int byteIdx = offset; + int valueIdx = 0; + int leftBits = 8; + int totalBits = 0; + + while (valueIdx < n) { + values[valueIdx] = 0; + while (totalBits < width) { + if (width - totalBits >= leftBits) { + values[valueIdx] = values[valueIdx] << leftBits; + values[valueIdx] = + values[valueIdx] | (((1L << leftBits) - 1) & (buf[byteIdx] & 0xFF)); + totalBits += leftBits; + byteIdx++; + leftBits = 8; + } else { + int t = width - totalBits; + values[valueIdx] = values[valueIdx] << t; + values[valueIdx] = + values[valueIdx] + | ((((1L << leftBits) - 1) & (buf[byteIdx] & 0xFF)) >>> (leftBits - t)); + leftBits -= t; + totalBits += t; + } + } + valueIdx++; + totalBits = 0; + } + } + /** * decode all values from 'buf' with specified offset and length decoded result will be saved in * array named 'values'. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java index 468bfcc55..b318627fe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java @@ -29,16 +29,27 @@ import java.util.Arrays; public class LongSprintzDecoder extends SprintzDecoder { + + private static final byte OPTIMAL_MODE_MARKER = 0; + private static final byte OPTIMAL_MODE_VERSION = 1; + LongPacker packer; LongFire firePred; private long preValue; - private final long[] currentBuffer; + private long[] currentBuffer; private long currentValue; private final String predictScheme = TSFileDescriptor.getInstance().getConfig().getSprintzPredictScheme(); + /** Whether we're decoding optimal mode (per-block pack size). */ + private Boolean optimalMode = null; + + /** Number of residuals in current block (for optimal mode). */ + private int currentBlockSize; + public LongSprintzDecoder() { super(); + Block_size = TSFileDescriptor.getInstance().getConfig().getSprintzBlockSize(); firePred = new LongFire(3); currentBuffer = new long[Block_size + 1]; reset(); @@ -46,8 +57,7 @@ public LongSprintzDecoder() { @Override public boolean hasNext(ByteBuffer buffer) throws IOException { - // int minLenth = Long.BYTES + 1; - return (isBlockReaded && currentCount < Block_size) || buffer.remaining() > 0; + return (isBlockReaded && currentCount < decodeSize) || buffer.remaining() > 0; } @Override @@ -56,48 +66,184 @@ public void reset() { preValue = 0; currentValue = 0; currentCount = 0; - Arrays.fill(currentBuffer, 0); + optimalMode = null; + if (currentBuffer != null) { + Arrays.fill(currentBuffer, 0); + } + } + + private void ensureOptimalModeDetermined(ByteBuffer in) throws IOException { + if (optimalMode != null) { + return; + } + if (in.remaining() < 2) { + return; + } + byte first = in.get(); + byte second = in.get(); + optimalMode = (first == OPTIMAL_MODE_MARKER && second == OPTIMAL_MODE_VERSION); + if (!optimalMode) { + in.position(in.position() - 2); + } } @Override protected void decodeBlock(ByteBuffer in) throws IOException { + ensureOptimalModeDetermined(in); + + if (Boolean.TRUE.equals(optimalMode)) { + decodeOptimalBlock(in); + } else { + decodeLegacyBlock(in); + } + isBlockReaded = true; + } + + private void decodeOptimalBlock(ByteBuffer in) throws IOException { + int packSize = in.get() & 0xFF; + if (packSize == 0) { + if (in.remaining() < 1) { + throw new IOException( + "Sprintz optimal block: need at least 1 byte after packSize=0"); + } + int next = in.get() & 0xFF; + if (next == 0) { + // Single-value block: [0][0][preValue 8 bytes], no RLE + if (in.remaining() < 8) { + throw new IOException( + "Sprintz optimal single-value block: need 8 bytes for preValue, have " + in.remaining()); + } + decodeSize = 1; + currentBlockSize = 0; + if (currentBuffer == null || currentBuffer.length < 1) { + currentBuffer = new long[33]; + } + currentBuffer[0] = in.getLong(); + return; + } + // RLE block: [0][size] format (legacy / backward compat) + decodeSize = next; + LongRleDecoder decoder = new LongRleDecoder(); + if (currentBuffer == null || currentBuffer.length < decodeSize) { + currentBuffer = new long[Math.max(decodeSize, 33)]; + } + for (int i = 0; i < decodeSize; i++) { + currentBuffer[i] = decoder.readLong(in); + } + currentBlockSize = 0; + return; + } + + packSize = Math.min(packSize, 32); + if (in.remaining() < 1 + 8) { + throw new IOException( + "Sprintz optimal block: need 9 bytes for bitWidth+preValue, have " + in.remaining()); + } bitWidth = ReadWriteForEncodingUtils.readIntLittleEndianPaddedOnBitWidth(in, 1); + preValue = in.getLong(); + decodeSize = packSize + 1; + currentBlockSize = packSize; + + int packedBytes = (packSize * bitWidth + 7) / 8; + if (in.remaining() < packedBytes) { + throw new IOException( + "Sprintz optimal block: need " + packedBytes + " bytes for packed data, have " + + in.remaining() + " (packSize=" + packSize + ", bitWidth=" + bitWidth + ")"); + } + + if (currentBuffer == null || currentBuffer.length < decodeSize) { + currentBuffer = new long[Math.max(decodeSize, 33)]; + } + currentBuffer[0] = preValue; + + long[] tmpBuffer = new long[packSize]; + packer = new LongPacker(bitWidth); + byte[] packcle = new byte[packedBytes]; + for (int i = 0; i < packedBytes; i++) { + packcle[i] = in.get(); + } + packer.unpackNValues(packcle, 0, packSize, tmpBuffer); + for (int i = 0; i < packSize; i++) { + currentBuffer[i + 1] = tmpBuffer[i]; + } + recalculate(currentBlockSize); + } + + private void decodeLegacyBlock(ByteBuffer in) throws IOException { + if (in.remaining() < 1) { + throw new IOException("Sprintz legacy block: no data remaining"); + } + bitWidth = ReadWriteForEncodingUtils.readIntLittleEndianPaddedOnBitWidth(in, 1); + // Encoder writes trailing RLE as [0][size] so bitWidth=0 means RLE block + if (bitWidth == 0) { + if (in.remaining() < 1) { + throw new IOException("Sprintz legacy RLE block: need 1 byte for size, have " + in.remaining()); + } + decodeSize = in.get() & 0xFF; + LongRleDecoder decoder = new LongRleDecoder(); + if (currentBuffer == null || currentBuffer.length < decodeSize) { + currentBuffer = new long[Math.max(decodeSize, Block_size + 1)]; + } + for (int i = 0; i < decodeSize; i++) { + currentBuffer[i] = decoder.readLong(in); + } + currentBlockSize = 0; + return; + } if ((bitWidth & (1 << 7)) != 0) { decodeSize = bitWidth & ~(1 << 7); LongRleDecoder decoder = new LongRleDecoder(); + if (currentBuffer == null || currentBuffer.length < decodeSize) { + currentBuffer = new long[Math.max(decodeSize, Block_size + 1)]; + } for (int i = 0; i < decodeSize; i++) { currentBuffer[i] = decoder.readLong(in); } - } else { - decodeSize = Block_size + 1; - preValue = in.getLong(); - currentBuffer[0] = preValue; - long[] tmpBuffer = new long[8]; - packer = new LongPacker(bitWidth); - byte[] packcle = new byte[bitWidth]; - for (int i = 0; i < bitWidth; i++) { - packcle[i] = in.get(); - } - packer.unpack8Values(packcle, 0, tmpBuffer); - for (int i = 0; i < 8; i++) currentBuffer[i + 1] = tmpBuffer[i]; - recalculate(); + currentBlockSize = 0; + return; } - isBlockReaded = true; + + if (in.remaining() < 8) { + throw new IOException( + "Sprintz legacy block: need 8 bytes for preValue, have " + in.remaining()); + } + decodeSize = Block_size + 1; + currentBlockSize = Block_size; + preValue = in.getLong(); + if (currentBuffer == null || currentBuffer.length < decodeSize) { + currentBuffer = new long[decodeSize]; + } + currentBuffer[0] = preValue; + + long[] tmpBuffer = new long[Block_size]; + packer = new LongPacker(bitWidth); + int packedBytes = (Block_size * bitWidth + 7) / 8; + byte[] packcle = new byte[packedBytes]; + for (int i = 0; i < packedBytes; i++) { + packcle[i] = in.get(); + } + packer.unpackNValues(packcle, 0, Block_size, tmpBuffer); + for (int i = 0; i < Block_size; i++) { + currentBuffer[i + 1] = tmpBuffer[i]; + } + recalculate(currentBlockSize); } - @Override - protected void recalculate() { - for (int i = 1; i <= Block_size; i++) { - if (currentBuffer[i] % 2 == 0) currentBuffer[i] = -currentBuffer[i] / 2; - else currentBuffer[i] = (currentBuffer[i] + 1) / 2; + private void recalculate(int blockSize) { + for (int i = 1; i <= blockSize; i++) { + if (currentBuffer[i] % 2 == 0) { + currentBuffer[i] = -currentBuffer[i] / 2; + } else { + currentBuffer[i] = (currentBuffer[i] + 1) / 2; + } } if (predictScheme.equals("delta")) { - for (int i = 1; i < currentBuffer.length; i++) { + for (int i = 1; i <= blockSize; i++) { currentBuffer[i] += currentBuffer[i - 1]; } } else if (predictScheme.equals("fire")) { firePred.reset(); - for (int i = 1; i <= Block_size; i++) { + for (int i = 1; i <= blockSize; i++) { long pred = firePred.predict(currentBuffer[i - 1]); long err = currentBuffer[i]; currentBuffer[i] = pred + err; @@ -108,13 +254,18 @@ protected void recalculate() { } } + @Override + protected void recalculate() { + recalculate(currentBlockSize > 0 ? currentBlockSize : Block_size); + } + @Override public long readLong(ByteBuffer buffer) { - if (!isBlockReaded) { + if (!isBlockReaded || currentCount >= decodeSize) { try { decodeBlock(buffer); } catch (IOException e) { - logger.error("Error occured when readInt with Sprintz Decoder.", e); + logger.error("Error occurred when readLong with Sprintz Decoder.", e); } } currentValue = currentBuffer[currentCount++]; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java index 2d91b4cf3..45f202efa 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java @@ -21,27 +21,35 @@ import org.apache.tsfile.encoding.bitpacking.LongPacker; import org.apache.tsfile.encoding.fire.LongFire; +import org.apache.tsfile.encoding.optimal.SprintzOptimalPackSize; import org.apache.tsfile.exception.encoding.TsFileEncodingException; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Vector; public class LongSprintzEncoder extends SprintzEncoder { - // bit packer - LongPacker packer; + private static final int OPTIMAL_CHUNK_MIN_SIZE = 32; + private static final byte OPTIMAL_MODE_MARKER = 0; + private static final byte OPTIMAL_MODE_VERSION = 1; - // Long Fire predictor + LongPacker packer; LongFire firePred; - - // we save all value in a list and calculate its bitwidth. protected Vector values; + /** For optimal mode: buffer to collect values before finding optimal pack size. */ + private final ArrayList chunkBuffer = new ArrayList<>(); + + /** For optimal mode: whether we've written the mode marker. */ + private boolean optimalModeMarkerWritten = false; + public LongSprintzEncoder() { super(); + Block_size = config.getSprintzBlockSize(); values = new Vector<>(); firePred = new LongFire(3); } @@ -50,16 +58,18 @@ public LongSprintzEncoder() { protected void reset() { super.reset(); values.clear(); + chunkBuffer.clear(); + optimalModeMarkerWritten = false; } @Override public int getOneItemMaxSize() { - return 1 + (1 + Block_size) * Long.BYTES; + return 1 + (1 + 32) * Long.BYTES; } @Override public long getMaxByteSize() { - return 1 + (1L + values.size()) * Long.BYTES; + return 1 + (1L + values.size() + chunkBuffer.size()) * Long.BYTES; } protected Long predict(Long value, Long preVlaue) throws TsFileEncodingException { @@ -75,7 +85,7 @@ protected Long predict(Long value, Long preVlaue) throws TsFileEncodingException if (pred <= 0) { pred = -2 * pred; } else { - pred = 2 * pred - 1; // TODO:overflow + pred = 2 * pred - 1; } return pred; } @@ -86,12 +96,13 @@ protected void bitPack() throws IOException { values.remove(0); this.bitWidth = ReadWriteForEncodingUtils.getLongMaxBitWidth(values); packer = new LongPacker(this.bitWidth); - byte[] bytes = new byte[bitWidth]; + int packedBytes = (Block_size * bitWidth + 7) / 8; + byte[] bytes = new byte[packedBytes]; long[] tmpBuffer = new long[Block_size]; for (int i = 0; i < Block_size; i++) { tmpBuffer[i] = values.get(i); } - packer.pack8Values(tmpBuffer, 0, bytes); + packer.packNValues(tmpBuffer, 0, Block_size, bytes); ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(bitWidth, byteCache, 1); byteCache.write(ByteBuffer.allocate(8).putLong(preValue).array()); byteCache.write(bytes, 0, bytes.length); @@ -108,33 +119,212 @@ protected Long fire(Long value, Long preValue) { return err; } + /** + * Encode a chunk with optimal pack size. Each block finds its own optimal pack size. + */ + private void encodeChunkWithOptimalPackSize(long[] originals, long[] residuals) throws IOException { + int n = residuals.length; + if (n == 0) { + return; + } + + int packSize = SprintzOptimalPackSize.findOptimalPackSize(residuals); + packSize = Math.max(1, Math.min(32, packSize)); + + int numPacks = (n + packSize - 1) / packSize; + + for (int p = 0; p < numPacks; p++) { + int start = p * packSize; + int end = Math.min(start + packSize, n); + int actualPackSize = end - start; + + long preValue = originals[start]; + long[] packResiduals = new long[actualPackSize]; + for (int i = 0; i < actualPackSize; i++) { + packResiduals[i] = residuals[start + i]; + } + + int packBitWidth = getLongArrayMaxBitWidth(packResiduals); + packBitWidth = Math.max(1, packBitWidth); + + packer = new LongPacker(packBitWidth); + int packedBytes = (actualPackSize * packBitWidth + 7) / 8; + byte[] bytes = new byte[packedBytes]; + packer.packNValues(packResiduals, 0, actualPackSize, bytes); + + byteCache.write(actualPackSize); + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(packBitWidth, byteCache, 1); + byteCache.write(ByteBuffer.allocate(8).putLong(preValue).array()); + byteCache.write(bytes, 0, bytes.length); + } + } + + private static int getLongArrayMaxBitWidth(long[] arr) { + int max = 1; + for (long num : arr) { + int bw = 64 - Long.numberOfLeadingZeros(Math.max(1, num)); + max = Math.max(max, bw); + } + return max; + } + @Override public void flush(ByteArrayOutputStream out) throws IOException { + if (config.isSprintzUseOptimalPackSize() && !chunkBuffer.isEmpty()) { + encodeRemainingChunkWithOptimal(); + } if (byteCache.size() > 0) { byteCache.writeTo(out); } if (!values.isEmpty()) { - int size = values.size(); - size |= (1 << 7); - ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(size, out, 1); - LongRleEncoder encoder = new LongRleEncoder(); - for (long val : values) { - encoder.encode(val, out); + int n = values.size(); + if (config.isSprintzUseOptimalPackSize()) { + // Encode trailing values as one optimal-style block, no RLE + long[] originals = new long[n]; + for (int i = 0; i < n; i++) { + originals[i] = values.get(i); + } + if (n == 1) { + // Single value: [0][0][preValue 8 bytes] so decoder treats packSize=0, next=0 as one value + out.write(0); + out.write(0); + out.write(ByteBuffer.allocate(8).putLong(originals[0]).array()); + } else { + long[] residuals = new long[n - 1]; + firePred.reset(); + long pre = originals[0]; + for (int i = 1; i < n; i++) { + try { + residuals[i - 1] = predict(originals[i], pre); + } catch (TsFileEncodingException e) { + throw new IOException(e); + } + pre = originals[i]; + } + int packBitWidth = getLongArrayMaxBitWidth(residuals); + packBitWidth = Math.max(1, packBitWidth); + int actualPackSize = n - 1; + int packedBytes = (actualPackSize * packBitWidth + 7) / 8; + byte[] bytes = new byte[packedBytes]; + packer = new LongPacker(packBitWidth); + packer.packNValues(residuals, 0, actualPackSize, bytes); + out.write(n - 1); + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(packBitWidth, out, 1); + out.write(ByteBuffer.allocate(8).putLong(originals[0]).array()); + out.write(bytes, 0, bytes.length); + } + } else { + // Legacy: [0][size] then RLE for trailing (decoder expects bitWidth=0 for this) + out.write(0); + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(n, out, 1); + LongRleEncoder encoder = new LongRleEncoder(); + for (long val : values) { + encoder.encode(val, out); + } + encoder.flush(out); } - encoder.flush(out); } reset(); } + private void encodeRemainingChunkWithOptimal() throws IOException { + if (chunkBuffer.size() < 2) { + values.addAll(chunkBuffer); + chunkBuffer.clear(); + return; + } + + int n = chunkBuffer.size(); + long[] originals = new long[n]; + for (int i = 0; i < n; i++) { + originals[i] = chunkBuffer.get(i); + } + + long[] residuals = new long[n - 1]; + firePred.reset(); + long pre = originals[0]; + for (int i = 1; i < n; i++) { + try { + residuals[i - 1] = predict(originals[i], pre); + } catch (TsFileEncodingException e) { + throw new IOException(e); + } + pre = originals[i]; + } + + if (!optimalModeMarkerWritten) { + byteCache.write(OPTIMAL_MODE_MARKER); + byteCache.write(OPTIMAL_MODE_VERSION); + optimalModeMarkerWritten = true; + } + encodeChunkWithOptimalPackSize(originals, residuals); + chunkBuffer.clear(); + } + @Override public void encode(long value, ByteArrayOutputStream out) { + if (config.isSprintzUseOptimalPackSize()) { + encodeOptimalMode(value, out); + return; + } + encodeLegacyMode(value, out); + } + + private void encodeOptimalMode(long value, ByteArrayOutputStream out) { + chunkBuffer.add(value); + + if (chunkBuffer.size() >= OPTIMAL_CHUNK_MIN_SIZE) { + try { + if (!optimalModeMarkerWritten) { + byteCache.write(OPTIMAL_MODE_MARKER); + byteCache.write(OPTIMAL_MODE_VERSION); + optimalModeMarkerWritten = true; + } + + int n = chunkBuffer.size(); + long[] originals = new long[n]; + for (int i = 0; i < n; i++) { + originals[i] = chunkBuffer.get(i); + } + + long[] residuals = new long[n - 1]; + firePred.reset(); + long pre = originals[0]; + for (int i = 1; i < n; i++) { + try { + residuals[i - 1] = predict(originals[i], pre); + } catch (TsFileEncodingException e) { + logger.error("Error in optimal Sprintz encoding", e); + throw new TsFileEncodingException("Sprintz optimal encoding failed", e); + } + pre = originals[i]; + } + + encodeChunkWithOptimalPackSize(originals, residuals); + chunkBuffer.clear(); + groupNum++; + if (groupNum == groupMax) { + // Write accumulated chunks to out but do NOT call full flush(): reset() would set + // optimalModeMarkerWritten=false and we'd write [0][1] again, which the decoder + // would misinterpret as RLE and desync (e.g. "need 20 bytes for packed data, have 5"). + byteCache.writeTo(out); + byteCache.reset(); + groupNum = 0; + } + } catch (IOException e) { + logger.error("Error occurred when encoding with optimal Sprintz", e); + throw new TsFileEncodingException("Sprintz optimal encoding failed", e); + } + } + } + + private void encodeLegacyMode(long value, ByteArrayOutputStream out) { if (!isFirstCached) { values.add(value); isFirstCached = true; return; - } else { - values.add(value); } + values.add(value); if (values.size() == Block_size + 1) { try { long pre = values.get(0); @@ -152,7 +342,8 @@ public void encode(long value, ByteArrayOutputStream out) { flush(out); } } catch (IOException e) { - logger.error("Error occured when encoding INT64 Type value with with Sprintz", e); + logger.error("Error occurred when encoding INT64 Type value with Sprintz", e); + throw new TsFileEncodingException("Sprintz legacy encoding failed", e); } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/SprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/SprintzEncoder.java index 4cdbe5590..e080cbf59 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/SprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/SprintzEncoder.java @@ -32,7 +32,7 @@ public abstract class SprintzEncoder extends Encoder { protected static final Logger logger = LoggerFactory.getLogger(SprintzEncoder.class); - // Segment block size to compress:8 + // Segment block size to compress: 8 (LongSprintzEncoder overrides from config) protected int Block_size = 8; // group size maximum diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java new file mode 100644 index 000000000..6c606439c --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding.optimal; + +/** + * Utility for finding optimal pack size for Sprintz bit-packing encoding using RMQ (Range Maximum + * Query) sparse table. Minimizes total storage cost: sum(pack_size * max_bitwidth_in_pack) + + * num_packs * BITS_PER_BLOCK_OVERHEAD. Each block in the encoder writes packSize(1 byte) + + * bitWidth(1 byte) + preValue(8 bytes) = 80 bits overhead, so we use 80 in the cost model. + */ +public class SprintzOptimalPackSize { + + private SprintzOptimalPackSize() {} + + /** + * Find optimal pack size for long values (e.g., Sprintz residuals) using RMQ-based cost + * minimization. + * + * @param values array of non-negative long values (e.g., after Sprintz predict transform) + * @return optimal pack size in range [1, n] + */ + public static int findOptimalPackSize(long[] values) { + int n = values.length; + if (n <= 0) { + return 1; + } + if (n < 8) { + return Math.max(1, n); + } + + // Build sparse table for RMQ on bit widths + int[] bitWidths = new int[n]; + long globalMax = 0; + for (int i = 0; i < n; i++) { + long value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + // Per-block overhead in encoder: 1 byte packSize + 1 byte bitWidth + 8 bytes preValue = 80 bits + final int bitsPerBlockOverhead = 80; + + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + + for (int i = 0; i < n; i++) { + st[0][i] = bitWidths[i]; + } + + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + int maxPackSize = Math.min(32, n); // encoder caps pack size at 32 + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; + long cost = 0; + + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + int k = log2[p]; + int maxBitWidth = + Math.max(st[k][start], st[k][end - (1 << k) + 1]); + cost += (long) p * maxBitWidth; + } + + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + int k = log2[r]; + int lastMaxBitWidth = + Math.max(st[k][lastStart], st[k][lastEnd - (1 << k) + 1]); + cost += (long) r * lastMaxBitWidth; + } + } + + cost += (long) m * bitsPerBlockOverhead; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + + return bestPackSize; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/exception/encoding/TsFileEncodingException.java b/java/tsfile/src/main/java/org/apache/tsfile/exception/encoding/TsFileEncodingException.java index 41a1ce4a4..3391a1226 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/exception/encoding/TsFileEncodingException.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/exception/encoding/TsFileEncodingException.java @@ -32,4 +32,8 @@ public class TsFileEncodingException extends TsFileRuntimeException { public TsFileEncodingException(String message) { super(message); } + + public TsFileEncodingException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java index 2576706b2..36f96d641 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java @@ -142,6 +142,8 @@ private void uncompressDataIfNecessary() throws IOException { if (lazyLoadPageData != null && (timeBuffer == null || valueBuffer == null)) { splitDataToTimeStampAndValue(lazyLoadPageData.uncompressPageData(pageHeader)); lazyLoadPageData = null; + valueDecoder.reset(); + timeDecoder.reset(); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/ALPTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/ALPTest.java new file mode 100644 index 000000000..833d78e41 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/ALPTest.java @@ -0,0 +1,1021 @@ +package org.apache.tsfile.encoding; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; + +public class ALPTest { + /** Written after block_size when using per-block submode (V5 int vs long pack-8 fallback). */ + private static final int ALP_OPTIMAL_V5_FORMAT_MAGIC = 0x41355635; + /** After min: {@link #ALP_V5_SUBMODE_INT_V2} then packSize + body; {@link #ALP_V5_SUBMODE_LONG_PACK8} then bit_width + long pack8. */ + private static final int ALP_V5_SUBMODE_INT_V2 = 0; + private static final int ALP_V5_SUBMODE_LONG_PACK8 = 1; + + static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv", "POI-lat.csv", + "POI-lon.csv", "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv"); //,"Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv" + + public static int bitWidth(int value) { + return 32 - Integer.numberOfLeadingZeros(value); + } + + public static int bitWidth(long value) { + return 64 - Long.numberOfLeadingZeros(value); + } + + public static void intToBytes(int srcNum, byte[] result, int pos, int width) { + int cnt = pos & 0x07; + int index = pos >> 3; + while (width > 0) { + int m = width + cnt >= 8 ? 8 - cnt : width; + width -= m; + int mask = 1 << (8 - cnt); + cnt += m; + byte y = (byte) (srcNum >>> width); + y = (byte) (y << (8 - cnt)); + mask = ~(mask - (1 << (8 - cnt))); + result[index] = (byte) (result[index] & mask | y); + srcNum = srcNum & ~(-1 << width); + if (cnt == 8) { + index++; + cnt = 0; + } + } + } + + public static int bytesToInt(byte[] result, int pos, int width) { + int ret = 0; + int cnt = pos & 0x07; + int index = pos >> 3; + while (width > 0) { + int m = width + cnt >= 8 ? 8 - cnt : width; + width -= m; + ret = ret << m; + byte y = (byte) (result[index] & (0xff >> cnt)); + y = (byte) ((y & 0xff) >>> (8 - cnt - m)); + ret = ret | (y & 0xff); + cnt += m; + if (cnt == 8) { + cnt = 0; + index++; + } + } + return ret; + } + + public static void boolToBytes(boolean value, byte[] result, int pos) { + int byteIndex = pos >> 3; + int bitOffset = pos & 0x07; + + if (value) { + result[byteIndex] |= (1 << (7 - bitOffset)); + } else { + result[byteIndex] &= ~(1 << (7 - bitOffset)); + } + } + + public static boolean bytesToBool(byte[] result, int pos) { + int byteIndex = pos >> 3; + int bitOffset = pos & 0x07; + + return (result[byteIndex] & (1 << (7 - bitOffset))) != 0; + } + + public static void longToBytes(long srcNum, byte[] result, int pos, int width) { + int cnt = pos & 0x07; + int index = pos >> 3; + + while (width > 0) { + int m = width + cnt >= 8 ? 8 - cnt : width; + width -= m; + int mask = 1 << (8 - cnt); + cnt += m; + byte y = (byte) (srcNum >>> width); + y = (byte) (y << (8 - cnt)); + mask = ~(mask - (1 << (8 - cnt))); + result[index] = (byte) (result[index] & mask | y); + srcNum = srcNum & ~(-1L << width); + if (cnt == 8) { + index++; + cnt = 0; + } + } + } + + public static long bytesToLong(byte[] result, int pos, int width) { + long ret = 0; + int cnt = pos & 0x07; + int index = pos >> 3; + while (width > 0) { + int m = width + cnt >= 8 ? 8 - cnt : width; + width -= m; + ret = ret << m; + byte y = (byte) (result[index] & (0xff >> cnt)); + y = (byte) ((y & 0xff) >>> (8 - cnt - m)); + ret = ret | (y & 0xff); + cnt += m; + if (cnt == 8) { + cnt = 0; + index++; + } + } + return ret; + } + + public static void pack8Values(int[] values, int offset, int width, int encode_pos, + byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + + while (valueIdx < 8 + offset) { + int buffer = 0; + int leftSize = 32; + + if (leftBit > 0) { + buffer |= (values[valueIdx] << (32 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + buffer |= (values[valueIdx] << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + + if (leftSize > 0 && valueIdx < 8 + offset) { + buffer |= (values[valueIdx] >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 4; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width) { + return; + } + } + } + + } + + public static void pack8Values( + long[] values, int offset, int width, int encode_pos, byte[] encoded_result) { + int bufIdx = 0; + int valueIdx = offset; + int leftBit = 0; + + while (valueIdx < 8 + offset) { + long buffer = 0; + int leftSize = 64; + + if (leftBit > 0) { + buffer |= (values[valueIdx] << (64 - leftBit)); + leftSize -= leftBit; + leftBit = 0; + valueIdx++; + } + + while (leftSize >= width && valueIdx < 8 + offset) { + buffer |= (values[valueIdx] << (leftSize - width)); + leftSize -= width; + valueIdx++; + } + if (leftSize > 0 && valueIdx < 8 + offset) { + buffer |= (values[valueIdx] >>> (width - leftSize)); + leftBit = width - leftSize; + } + + for (int j = 0; j < 8; j++) { + encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF); + encode_pos++; + bufIdx++; + if (bufIdx >= width * 8 / 8) { + return; + } + } + } + } + + public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) { + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < 8) { + result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width)); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static void unpack8Values( + byte[] encoded, int offset, int width, long[] result_list, int result_offset) { + int byteIdx = offset; + long buffer = 0; + int totalBits = 0; + int valueIdx = 0; + + while (valueIdx < 8) { + while (totalBits < width) { + buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF); + byteIdx++; + totalBits += 8; + } + + while (totalBits >= width && valueIdx < 8) { + result_list[result_offset + valueIdx] = buffer >>> (totalBits - width); + valueIdx++; + totalBits -= width; + buffer = buffer & ((1L << totalBits) - 1); + } + } + } + + public static int bitPacking(int[] numbers, int bit_width, int encode_pos, + byte[] encoded_result, int num_values) { + int block_num = num_values / 8; + int remainder = num_values % 8; + + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + encode_pos *= 8; + + for (int i = 0; i < remainder; i++) { + intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width); + encode_pos += bit_width; + } + + return (encode_pos + 7) / 8; + } + + public static int bitPacking(long[] numbers, int bit_width, int encode_pos, + byte[] encoded_result, int num_values) { + int block_num = num_values / 8; + int remainder = num_values % 8; + + for (int i = 0; i < block_num; i++) { + pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result); + encode_pos += bit_width; + } + + encode_pos *= 8; + + for (int i = 0; i < remainder; i++) { + longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width); + encode_pos += bit_width; + } + + return (encode_pos + 7) / 8; + } + + public static int decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) { + int block_num = num_values / 8; + int remainder = num_values % 8; + + for (int i = 0; i < block_num; i++) { + unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8); + decode_pos += bit_width; + } + + decode_pos *= 8; + + for (int i = 0; i < remainder; i++) { + result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width); + decode_pos += bit_width; + } + + return (decode_pos + 7) / 8; + } + + public static int decodeBitPacking( + byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) { + int block_num = num_values / 8; + int remainder = num_values % 8; + + for (int i = 0; i < block_num; i++) { + unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8); + decode_pos += bit_width; + } + + decode_pos *= 8; + + for (int i = 0; i < remainder; i++) { + result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width); + decode_pos += bit_width; + } + + return (decode_pos + 7) / 8; + } + + public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer); + } + + public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 56); + cur_byte[encode_pos + 1] = (byte) (integer >> 48); + cur_byte[encode_pos + 2] = (byte) (integer >> 40); + cur_byte[encode_pos + 3] = (byte) (integer >> 32); + cur_byte[encode_pos + 4] = (byte) (integer >> 24); + cur_byte[encode_pos + 5] = (byte) (integer >> 16); + cur_byte[encode_pos + 6] = (byte) (integer >> 8); + cur_byte[encode_pos + 7] = (byte) (integer); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + public static long bytes2Long(byte[] encoded, int start, int num) { + long value = 0; + + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int encode_pos, + int max_decimal, + byte[] encoded_result) { + int e = 0; + int f = 0; + + int base = block_index * block_size; + + e = 17; + f = 17 - max_decimal; + + double[] testValues = new double[remainder]; + for (int i = 0; i < testValues.length; i++) { + testValues[i] = data[base + (i * remainder / testValues.length)]; + } + + outer_loop: for (int i = 0; i < 19; i++) { + for (int j = i; j >= 0; j--) { + boolean valid = true; + + for (double value : testValues) { + long encodedValue = Math.round(value * power(10, i) / power(10, j)); + + double decodedValue = encodedValue / power(10, i) * power(10, j); + if (Math.abs(value - decodedValue) > Math.pow(10, -max_decimal)) { + valid = false; + break; + } + } + + if (valid) { + e = i; + f = j; + + break outer_loop; + } + } + } + + intByte2Bytes(e, encode_pos, encoded_result); + encode_pos += 1; + + intByte2Bytes(f, encode_pos, encoded_result); + encode_pos += 1; + + long[] data_long = new long[remainder]; + + long min_data = Long.MAX_VALUE; + long max_data = Long.MIN_VALUE; + + for (int i = 0; i < remainder; i++) { + data_long[i] = Math.round(data[base + i] * power(10, e) / power(10, f)); + if (data_long[i] < min_data) { + min_data = data_long[i]; + } + if (data_long[i] > max_data) { + max_data = data_long[i]; + } + } + + long2Bytes(min_data, encode_pos, encoded_result); + encode_pos += 8; + + for (int i = 0; i < remainder; i++) { + data_long[i] -= min_data; + } + long max_diff = max_data - min_data; + int bit_width = bitWidth(max_diff); + + intByte2Bytes(bit_width, encode_pos, encoded_result); + encode_pos += 1; + + encode_pos = bitPacking(data_long, bit_width, encode_pos, encoded_result, remainder); + return encode_pos; + + } + + public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder, + int encode_pos, double[] data) { + int e = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + + int f = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + + long min_data = bytes2Long(encoded_result, encode_pos, 8); + encode_pos += 8; + + int bit_width = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + + long[] data_long = new long[remainder]; + + encode_pos = decodeBitPacking(encoded_result, encode_pos, bit_width, remainder, data_long); + + for (int i = 0; i < remainder; i++) { + data_long[i] += min_data; + data[block_index * block_size + i] = data_long[i] * power(10, f) / power(10, e); + } + + return encode_pos; + } + + /** + * Same ALP block as {@link #BlockEncoder} but bit-packing uses + * {@link AllNo8PacksizeOptimal#findOptimalPackSizeallV5(int[])} per block and + * {@link AllNo8PacksizeOptimal#encodeBitPackingV2(int[], int[], int)} when all deltas fit in {@code int}. + * Otherwise falls back to the same long pack-8 layout as {@link #BlockEncoder}. + *

+ * With {@link #ALP_OPTIMAL_V5_FORMAT_MAGIC} in the stream header, after min: 1-byte submode + * ({@link #ALP_V5_SUBMODE_INT_V2} or {@link #ALP_V5_SUBMODE_LONG_PACK8}), then either V2 payload or bit_width + pack8. + */ + public static int BlockEncoderOptimalPackV5(double[] data, int block_index, int block_size, int remainder, + int encode_pos, int max_decimal, byte[] encoded_result) { + int e = 0; + int f = 0; + int base = block_index * block_size; + e = 17; + f = 17 - max_decimal; + + double[] testValues = new double[remainder]; + for (int i = 0; i < testValues.length; i++) { + testValues[i] = data[base + (i * remainder / testValues.length)]; + } + + outer_loop: + for (int i = 0; i < 19; i++) { + for (int j = i; j >= 0; j--) { + boolean valid = true; + for (double value : testValues) { + long encodedValue = Math.round(value * power(10, i) / power(10, j)); + double decodedValue = encodedValue / power(10, i) * power(10, j); + if (Math.abs(value - decodedValue) > Math.pow(10, -max_decimal)) { + valid = false; + break; + } + } + if (valid) { + e = i; + f = j; + break outer_loop; + } + } + } + + intByte2Bytes(e, encode_pos, encoded_result); + encode_pos += 1; + intByte2Bytes(f, encode_pos, encoded_result); + encode_pos += 1; + + long[] data_long = new long[remainder]; + long min_data = Long.MAX_VALUE; + long max_data = Long.MIN_VALUE; + for (int i = 0; i < remainder; i++) { + data_long[i] = Math.round(data[base + i] * power(10, e) / power(10, f)); + if (data_long[i] < min_data) { + min_data = data_long[i]; + } + if (data_long[i] > max_data) { + max_data = data_long[i]; + } + } + + long2Bytes(min_data, encode_pos, encoded_result); + encode_pos += 8; + + for (int i = 0; i < remainder; i++) { + data_long[i] -= min_data; + } + + boolean deltasFitInt = true; + for (int i = 0; i < remainder; i++) { + long v = data_long[i]; + if (v != (int) v) { + deltasFitInt = false; + break; + } + } + + if (!deltasFitInt) { + long maxDelta = 0L; + for (int i = 0; i < remainder; i++) { + if (data_long[i] > maxDelta) { + maxDelta = data_long[i]; + } + } + int bit_width = bitWidth(maxDelta); + intByte2Bytes(ALP_V5_SUBMODE_LONG_PACK8, encode_pos, encoded_result); + encode_pos += 1; + intByte2Bytes(bit_width, encode_pos, encoded_result); + encode_pos += 1; + encode_pos = bitPacking(data_long, bit_width, encode_pos, encoded_result, remainder); + return encode_pos; + } + + int[] asInt = new int[remainder]; + for (int i = 0; i < remainder; i++) { + asInt[i] = (int) data_long[i]; + } + + int packSize = AllNo8PacksizeOptimal.findOptimalPackSizeallV5(asInt); + int numGroups = (remainder + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + for (int g = 0; g < numGroups; g++) { + int start = g * packSize; + int end = Math.min(start + packSize, remainder); + int maxv = 0; + for (int i = start; i < end; i++) { + if (asInt[i] > maxv) { + maxv = asInt[i]; + } + } + bitWidths[g] = 64 - Long.numberOfLeadingZeros(Math.max(1L, (long) maxv)); + } + + byte[] body = AllNo8PacksizeOptimal.encodeBitPackingV2(asInt, bitWidths, packSize); + int need = encode_pos + 1 + 4 + 4 + body.length; + if (need > encoded_result.length) { + throw new IllegalStateException("encoded_result too small: need " + need + " have " + encoded_result.length); + } + intByte2Bytes(ALP_V5_SUBMODE_INT_V2, encode_pos, encoded_result); + encode_pos += 1; + int2Bytes(packSize, encode_pos, encoded_result); + encode_pos += 4; + int2Bytes(body.length, encode_pos, encoded_result); + encode_pos += 4; + System.arraycopy(body, 0, encoded_result, encode_pos, body.length); + encode_pos += body.length; + return encode_pos; + } + + public static int BlockDecoderOptimalPackV5(byte[] encoded_result, int block_index, int block_size, int remainder, + int encode_pos, double[] data, boolean submodeAfterMin) { + int e = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + int f = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + long min_data = bytes2Long(encoded_result, encode_pos, 8); + encode_pos += 8; + + long[] data_long = new long[remainder]; + if (submodeAfterMin) { + int mode = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + if (mode == ALP_V5_SUBMODE_LONG_PACK8) { + int bit_width = bytes2Integer(encoded_result, encode_pos, 1); + encode_pos += 1; + encode_pos = decodeBitPacking(encoded_result, encode_pos, bit_width, remainder, data_long); + } else { + int packSize = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + int bodyLen = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + byte[] body = Arrays.copyOfRange(encoded_result, encode_pos, encode_pos + bodyLen); + encode_pos += bodyLen; + int[] decoded = AllNo8PacksizeOptimal.decodeBitPackingV2Auto(body, packSize, remainder); + for (int i = 0; i < remainder; i++) { + data_long[i] = (long) decoded[i]; + } + } + } else { + int packSize = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + int bodyLen = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + byte[] body = Arrays.copyOfRange(encoded_result, encode_pos, encode_pos + bodyLen); + encode_pos += bodyLen; + int[] decoded = AllNo8PacksizeOptimal.decodeBitPackingV2Auto(body, packSize, remainder); + for (int i = 0; i < remainder; i++) { + data_long[i] = (long) decoded[i]; + } + } + + for (int i = 0; i < remainder; i++) { + data_long[i] += min_data; + data[block_index * block_size + i] = data_long[i] * power(10, f) / power(10, e); + } + return encode_pos; + } + + public static int EncoderOptimalPackV5(double[] data, int block_size, int max_decimal, byte[] encoded_result) { + int data_length = data.length; + int encode_pos = 0; + int2Bytes(data_length, encode_pos, encoded_result); + encode_pos += 4; + int2Bytes(block_size, encode_pos, encoded_result); + encode_pos += 4; + int2Bytes(ALP_OPTIMAL_V5_FORMAT_MAGIC, encode_pos, encoded_result); + encode_pos += 4; + int num_blocks = data_length / block_size; + int remainder = data_length % block_size; + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockEncoderOptimalPackV5(data, i, block_size, block_size, encode_pos, max_decimal, + encoded_result); + } + if (remainder > 0) { + encode_pos = BlockEncoderOptimalPackV5(data, num_blocks, block_size, remainder, encode_pos, max_decimal, + encoded_result); + } + return encode_pos; + } + + public static double[] DecoderOptimalPackV5(byte[] encoded_result) { + int encode_pos = 0; + int data_length = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + int block_size = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + boolean submodeAfterMin = false; + if (encoded_result.length >= encode_pos + 4) { + int magic = bytes2Integer(encoded_result, encode_pos, 4); + if (magic == ALP_OPTIMAL_V5_FORMAT_MAGIC) { + submodeAfterMin = true; + encode_pos += 4; + } + } + int num_blocks = data_length / block_size; + int remainder = data_length % block_size; + double[] data = new double[data_length]; + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockDecoderOptimalPackV5(encoded_result, i, block_size, block_size, encode_pos, data, + submodeAfterMin); + } + if (remainder > 0) { + encode_pos = BlockDecoderOptimalPackV5(encoded_result, num_blocks, block_size, remainder, encode_pos, data, + submodeAfterMin); + } + return data; + } + + public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) { + int data_length = data.length; + int encode_pos = 0; + + int2Bytes(data_length, encode_pos, encoded_result); + encode_pos += 4; + + int2Bytes(block_size, encode_pos, encoded_result); + encode_pos += 4; + + int num_blocks = data_length / block_size; + + int remainder = data_length % block_size; + + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, max_decimal, encoded_result); + } + + if (remainder > 0) { + encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, + max_decimal, encoded_result); + } + + return encode_pos; + } + + public static double[] Decoder(byte[] encoded_result) { + int encode_pos = 0; + + int data_length = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + + int block_size = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + + int num_blocks = data_length / block_size; + int remainder = data_length % block_size; + + double[] data = new double[data_length]; + + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data); + } + + if (remainder > 0) { + encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, encode_pos, data); + } + + return data; + } + + public static double power(int base, int exponent) { + double result = 1; + for (int i = 0; i < exponent; i++) { + result *= base; + } + return result; + } + + public static int getDecimalPrecision(String str) { + int decimalIndex = str.indexOf("."); + + if (decimalIndex == -1) { + return 0; + } + + return str.substring(decimalIndex + 1).length(); + } + + public static String extractFileName(String path) { + if (path == null || path.isEmpty()) { + return ""; + } + + File file = new File(path); + String fileName = file.getName(); + + int dotIndex = fileName.lastIndexOf('.'); + + if (dotIndex == -1 || dotIndex == 0) { + return fileName; + } + + return fileName.substring(0, dotIndex); + } + + @Test + public void test0() throws IOException { + // base dirs (adjust to your environment) + String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/"; + String input_parent_dir = parent_dir + "ElfTestData_camel/"; + String output_parent_dir = parent_dir + "output_ALP/"; + + int block_size = 1024; + int repeatTime = 50; + + File outputDir = new File(output_parent_dir); + if (!outputDir.exists()) outputDir.mkdirs(); + + File directory = new File(input_parent_dir); + File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv")); + + if (csvFiles == null) return; + + for (File file : csvFiles) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + String datasetName = extractFileName(file.toString()); + System.out.println("Processing dataset: " + datasetName); + + // per-dataset output file (one CSV per dataset) + String Output = output_parent_dir + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + // load input values + InputStream inputStream = Files.newInputStream(file.toPath()); + CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); + ArrayList data1 = new ArrayList<>(); + + int max_decimal = 0; + while (loader.readRecord()) { + String f_str = loader.getValues()[0]; + if (f_str == null || f_str.isEmpty()) continue; + int cur_decimal = getDecimalPrecision(f_str); + if (cur_decimal > max_decimal) max_decimal = cur_decimal; + data1.add(Double.valueOf(f_str)); + } + inputStream.close(); + + if (max_decimal > 17) max_decimal = 17; + + double[] data1_arr = new double[data1.size()]; + for (int i = 0; i < data1.size(); i++) data1_arr[i] = data1.get(i); + + // allocate output buffer (conservative estimate) + byte[] encoded_result = new byte[data1.size() * 8 + 1024]; + + long encodeTime = 0; + long decodeTime = 0; + double compressed_size = 0; + + int length = 0; + + // measure encoding (repeat and average) + long s = System.nanoTime(); + for (int repeat = 0; repeat < repeatTime; repeat++) { + length = Encoder(data1_arr, block_size, max_decimal, encoded_result); + } + long e = System.nanoTime(); + encodeTime = ((e - s) / repeatTime); // average nanoseconds + compressed_size = length; // bytes + + // measure decoding (repeat and average) + double[] data2_arr_decoded; + s = System.nanoTime(); + for (int repeat = 0; repeat < repeatTime; repeat++) { + data2_arr_decoded = Decoder(encoded_result); + } + e = System.nanoTime(); + decodeTime = ((e - s) / repeatTime); + + // convert times to throughput MB/s like BPTest: (points * 8000L) / time_ns + double encodeThroughput = 0.0; + double decodeThroughput = 0.0; + if (encodeTime > 0) encodeThroughput = (double) (data1.size() * 8000L) / (double) encodeTime; + if (decodeTime > 0) decodeThroughput = (double) (data1.size() * 8000L) / (double) decodeTime; + + double ratio = 0.0; + if (data1.size() > 0) { + ratio = compressed_size / (double) (data1.size() * Long.BYTES); + } + + String[] record = { + file.toString(), + "ALP", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(data1.size()), + String.valueOf((long) compressed_size), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Wrote results for " + datasetName + " -> " + Output); + System.out.println("Encoding throughput (MB/s): " + encodeThroughput); + System.out.println("Decoding throughput (MB/s): " + decodeThroughput); + System.out.println("Compressed size (bytes): " + compressed_size + " ratio: " + ratio); + } + } + + @Test + public void test1_optimalPackV5() throws IOException { + String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/"; + String input_parent_dir = parent_dir + "ElfTestData_camel/"; + String output_parent_dir = parent_dir + "output_ALP_optimal_v5/"; + + int block_size = 1024; + int repeatTime = 50; + + File outputDir = new File(output_parent_dir); + if (!outputDir.exists()) outputDir.mkdirs(); + + File directory = new File(input_parent_dir); + File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv")); + + if (csvFiles == null) return; + + for (File file : csvFiles) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + String datasetName = extractFileName(file.toString()); + System.out.println("Processing dataset (ALP+V5pack): " + datasetName); + + String Output = output_parent_dir + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + InputStream inputStream = Files.newInputStream(file.toPath()); + CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); + ArrayList data1 = new ArrayList<>(); + + int max_decimal = 0; + while (loader.readRecord()) { + String f_str = loader.getValues()[0]; + if (f_str == null || f_str.isEmpty()) continue; + int cur_decimal = getDecimalPrecision(f_str); + if (cur_decimal > max_decimal) max_decimal = cur_decimal; + data1.add(Double.valueOf(f_str)); + } + inputStream.close(); + + if (max_decimal > 17) max_decimal = 17; + + double[] data1_arr = new double[data1.size()]; + for (int i = 0; i < data1.size(); i++) data1_arr[i] = data1.get(i); + + byte[] encoded_result = new byte[Math.max(1 << 20, data1.size() * 32 + 65536)]; + + long encodeTime = 0; + long decodeTime = 0; + double compressed_size = 0; + int length = 0; + + long s = System.nanoTime(); + for (int repeat = 0; repeat < repeatTime; repeat++) { + length = EncoderOptimalPackV5(data1_arr, block_size, max_decimal, encoded_result); + } + long e = System.nanoTime(); + encodeTime = ((e - s) / repeatTime); + compressed_size = length; + + double[] data2_arr_decoded; + s = System.nanoTime(); + for (int repeat = 0; repeat < repeatTime; repeat++) { + data2_arr_decoded = DecoderOptimalPackV5(encoded_result); + } + e = System.nanoTime(); + decodeTime = ((e - s) / repeatTime); + + double encodeThroughput = 0.0; + double decodeThroughput = 0.0; + if (encodeTime > 0) encodeThroughput = (double) (data1.size() * 8000L) / (double) encodeTime; + if (decodeTime > 0) decodeThroughput = (double) (data1.size() * 8000L) / (double) decodeTime; + + double ratio = 0.0; + if (data1.size() > 0) { + ratio = compressed_size / (double) (data1.size() * Long.BYTES); + } + + String[] record = { + file.toString(), + "ALP+V5pack", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(data1.size()), + String.valueOf((long) compressed_size), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Wrote results for " + datasetName + " -> " + Output); + System.out.println("Encoding throughput (MB/s): " + encodeThroughput); + System.out.println("Decoding throughput (MB/s): " + decodeThroughput); + System.out.println("Compressed size (bytes): " + compressed_size + " ratio: " + ratio); + } + } + +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java index ded951ca9..de7acfa9f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -1,26 +1,48 @@ -package org.apache.iotdb.tsfile.encoding; +package org.apache.tsfile.encoding; import com.csvreader.CsvReader; import com.csvreader.CsvWriter; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assume; import org.junit.Test; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; -import static org.apache.commons.lang3.ObjectUtils.min; +// import static org.apache.commons.lang3.ObjectUtils.min; public class AllNo8PacksizeOptimal { static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv", "POI-lat.csv", "POI-lon.csv", "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv"); //,"Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv" + + static final List NO_TIME_SERIES_FILES = Arrays.asList("Food-price.csv", "electric_vehicle_charging.csv", "POI-lat.csv", "POI-lon.csv", "Blockchain-tr.csv", + "SSD-bench.csv", "City-lat.csv","City-lon.csv"); + private static final int CHUNK_SIZE = 1024; public static int getCount(long long1, int mask) { @@ -358,6 +380,33 @@ public static int[] decodeBitPackingV2(byte[] encodedData, int[] originalBitWidt return decodedArray; } + /** + * Decode a {@link #encodeBitPackingV2} payload without supplying bitWidths (uses widths read from the bit stream). + */ + public static int[] decodeBitPackingV2Auto(byte[] encodedData, int pack_size, int n) { + int totalGroups = (n + pack_size - 1) / pack_size; + int[] decodedArray = new int[n]; + BitReaderV2 bitReader = new BitReaderV2(encodedData); + int bitsForBitWidth = bitReader.readBits(6); + int[] decodedBitWidths = new int[totalGroups]; + for (int group = 0; group < totalGroups; group++) { + decodedBitWidths[group] = bitReader.readBits(bitsForBitWidth); + } + for (int group = 0; group < totalGroups; group++) { + int startIndex = group * pack_size; + int bitWidth = decodedBitWidths[group]; + int valuesInGroup = Math.min(pack_size, n - startIndex); + if (valuesInGroup <= 0) { + break; + } + for (int i = 0; i < valuesInGroup; i++) { + int idx = startIndex + i; + decodedArray[idx] = bitReader.readBits(bitWidth); + } + } + return decodedArray; + } + // 改进的BitWriterV2类 static class BitWriterV2 { private ByteArrayOutputStream buffer; @@ -781,7 +830,90 @@ public static int findOptimalPackSizeallV2(int[] values) { return bestPackSize; } - // 结合RMQ + + // 遍历所有情况 n^2 for sort + public static int findOptimalPackSizeallForSort(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算全局最大位宽和z值（以bit为单位） + int globalMax = 0; + for (int value : values) { + if (value > globalMax) { + globalMax = value; + } + } + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 枚举所有可能的pack_size + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + + // 限制最大pack_size，避免性能问题 + int maxPackSize = n; // 可以调整这个限制 + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + int r = n - (m - 1) * p; // 最后一个pack的大小 + + long cost = 0; + + // 计算前m-1个pack的成本（bit为单位） + for (int i = 0; i < m - 1; i++) { + int start = i * p; + // int end = start + p - 1; + + // 遍历当前pack的所有值，找出最大位宽 + int maxBitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[start])); + + // for (int j = start; j <= end; j++) { + // int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + // if (bitWidth > maxBitWidth) { + // maxBitWidth = bitWidth; + // } + // } + + // 直接计算bit数：p个值，每个值需要maxBitWidth位 + long bitsNeeded = (long) p * maxBitWidth; + cost += bitsNeeded; + } + + // 计算最后一个pack的成本 + if (m > 0 && r > 0) { + int lastStart = (m - 1) * p; + // int lastEnd = n - 1; + + int lastMaxBitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[lastStart])); + + // for (int j = lastStart; j <= lastEnd; j++) { + // int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[j])); + // if (bitWidth > lastMaxBitWidth) { + // lastMaxBitWidth = bitWidth; + // } + // } + + long bitsNeeded = (long) r * lastMaxBitWidth; + cost += bitsNeeded; + } + + // 加上位宽信息的存储成本（每个pack需要z位） + cost += (long) m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + +// if(bestPackSize == 1) + + + return bestPackSize; + } + + + // 结合RMQ + 无剪枝 public static int findOptimalPackSizeallV3(int[] values) { int n = values.length; if (n < 8) return n; @@ -872,6 +1004,86 @@ public static int findOptimalPackSizeallV3(int[] values) { return bestPackSize; } + + // 结合Sort的剪枝（去掉RMQ） + public static int findOptimalPackSizeallV3ForSort(int[] values) { + int n = values.length; + if (n < 8) return n; + + // 计算位宽数组 + int[] bitWidths = new int[n]; + int globalMax = 0; + for (int i = 0; i < n; i++) { + int value = values[i]; + if (value > globalMax) { + globalMax = value; + } + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + } + + int bitWidthGlobal = 64 - Long.numberOfLeadingZeros(Math.max(1, globalMax)); + int z = (int) Math.ceil(Math.log(bitWidthGlobal + 1) / Math.log(2)); + + // 构建稀疏表（RMQ） + int logN = 32 - Integer.numberOfLeadingZeros(n); // log2(n) + + + + // 预计算log2表 + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + // 枚举所有可能的pack_size + int bestPackSize = 1; + long bestCost = Long.MAX_VALUE; + int maxPackSize = n; + + for (int p = 1; p <= maxPackSize; p++) { + int m = (n + p - 1) / p; // ceil(n/p) + long cost = 0; + + // 计算前m-1个pack的成本 + for (int i = 0; i < m - 1; i++) { + int start = i * p; + // int end = start + p - 1; + + + int maxBitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[start])); + + cost += (long) p * maxBitWidth; + } + + // 计算最后一个pack的成本 + if (m > 0) { + int lastStart = (m - 1) * p; + // int lastEnd = n - 1; + int r = n - lastStart; + + if (r > 0) { + // int k = log2[r]; + int lastMaxBitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, values[lastStart])); + cost += (long) r * lastMaxBitWidth; + } + } + + // 加上位宽信息的存储成本 + cost += (long) m * z; + + if (cost < bestCost) { + bestCost = cost; + bestPackSize = p; + } + } + +// if (bestPackSize == 1) +// System.out.println(bestCost); + + return bestPackSize; + } + + // 结合RMQ+剪枝（保证正确性的高效版本） public static int findOptimalPackSizeallV4(int[] values) { int n = values.length; @@ -1259,6 +1471,90 @@ public static int findOptimalPackSizeallV5(int[] values) { return bestPackSize; } + /** + * Same structure as {@link #findOptimalPackSizeallV5(int[])}: RMQ max bit-width per segment + {@link #PREV_ARRAY} + * pruning, but (1) input is zigzag-unsigned magnitudes as {@code long[]}, and (2) per-pack metadata cost is + * {@code 8 * m} bits (one byte per pack, as in {@code CuSZpCpuTest.encodePlain1D}), instead of {@code m * z}. + */ + public static int findOptimalPackSizeCuSZpMeta8Bits(long[] valuesUnsigned) { + int n = valuesUnsigned.length; + if (n < 8) { + return n; + } + + int[] bitWidths = new int[n]; + for (int i = 0; i < n; i++) { + long v = valuesUnsigned[i]; + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1L, v)); + } + + int logN = 32 - Integer.numberOfLeadingZeros(n); + int[][] st = new int[logN][n]; + System.arraycopy(bitWidths, 0, st[0], 0, n); + for (int k = 1; k < logN; k++) { + int step = 1 << (k - 1); + for (int i = 0; i + (1 << k) <= n; i++) { + st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); + } + } + + int[] log2 = new int[n + 1]; + for (int i = 2; i <= n; i++) { + log2[i] = log2[i / 2] + 1; + } + + long[] cost = new long[n + 1]; + boolean[] isIncreased = new boolean[n + 1]; + long bestCost = Long.MAX_VALUE; + int bestPackSize = n; + + for (int p = 1; p <= n; p++) { + int prev = p < PREV_ARRAY.length ? PREV_ARRAY[p] : 0; + + if (prev != 0 && isIncreased[prev]) { + isIncreased[p] = true; + continue; + } + + int m = (n + p - 1) / p; + long currentCost = 0; + + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + int kk = log2[p]; + int maxBitWidth = Math.max(st[kk][start], st[kk][end - (1 << kk) + 1]); + currentCost += (long) p * maxBitWidth; + } + + if (m > 0) { + int lastStart = (m - 1) * p; + int lastEnd = n - 1; + int r = n - lastStart; + if (r > 0) { + int kk = log2[r]; + int lastMaxBitWidth = Math.max(st[kk][lastStart], st[kk][lastEnd - (1 << kk) + 1]); + currentCost += (long) r * lastMaxBitWidth; + } + } + + currentCost += (long) m * 8L; + cost[p] = currentCost; + + if (prev != 0 && currentCost > cost[prev]) { + isIncreased[p] = true; + continue; + } + + if (currentCost < bestCost) { + bestCost = currentCost; + bestPackSize = p; + } + } + + return bestPackSize; + } + public static int testFindOptimalPackSizeallV5(int[] values, int[] filters_count) { int n = values.length; if (n < 8) return n; @@ -2346,7 +2642,8 @@ public static int[] sprintzDecode(int[] encodedData) { - public static void main(String[] args) throws IOException { + @Test + public void OptimalPackSizeBPAllNo8Test() throws IOException { System.out.println("\nPerformance Testing..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_all_no8"; @@ -2392,7 +2689,7 @@ public static void main(String[] args) throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = 500; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -3741,6 +4038,10 @@ public void BPTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); + Assume.assumeTrue( + "Skip BPTest: dataset directory missing: " + directory, + dir.exists() && dir.isDirectory() + ); for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -3816,7 +4117,7 @@ public void BPTest() throws IOException { long startTime = System.nanoTime(); // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = 8; + int pack_size = 16; // 确保pack_size至少为1 pack_size = Math.max(1, pack_size); @@ -3879,23 +4180,27 @@ public void BPTest() throws IOException { } } - /// vary pack size - @Test - public void OptimalPackSizeN2Test() throws IOException { - System.out.println("\nPerformance Testing..."); + public void Simple8bTest() throws IOException { + System.out.println("\nPerformance Testing (Simple8b)..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_N2_all_no8"; - File outputDir = new File(outputDirstr); + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Simple8b"; - if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); - for (File file : Objects.requireNonNull(dir.listFiles())) { + Assume.assumeTrue( + "Skip Simple8bTest: dataset directory missing: " + directory, + dir.exists() && dir.isDirectory() + ); + File outputDir = new File(outputDirstr); + if (!outputDir.exists()) outputDir.mkdir(); + + for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; System.out.println(file.getName()); - String Output = outputDirstr + "/" + file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(output, ',', StandardCharsets.UTF_8); String[] head = { "Input Direction", @@ -3907,6 +4212,7 @@ public void OptimalPackSizeN2Test() throws IOException { "Compression Ratio" }; writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); List numbers = new ArrayList<>(); List decimalPlaces = new ArrayList<>(); @@ -3928,14 +4234,11 @@ public void OptimalPackSizeN2Test() throws IOException { } } } - int time_of_repeat = 200; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - // 分批处理，每1024个元素一批 int batchSize = 1024; List batches = new ArrayList<>(); - for (int i = 0; i < numbers.size(); i += batchSize) { int end = Math.min(numbers.size(), i + batchSize); List batch = numbers.subList(i, end); @@ -3943,22 +4246,498 @@ public void OptimalPackSizeN2Test() throws IOException { batches.add(scaledBatch); } - // 计算总长度并拼接所有批次的结果 int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - + int[] scaledIntsAll = new int[totalLength]; int currentIndex = 0; for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + System.arraycopy(batch, 0, scaledIntsAll, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + // correctness check (exclude from perf timing) + int[] roundTrip = simple8bRoundTripInts(scaledIntsAll); + if (!Arrays.equals(scaledIntsAll, roundTrip)) { + throw new AssertionError("Simple8b round-trip mismatch: " + file.getName()); + } + + int timeOfRepeat = 50; + long modelCostBits = 0; + long modelTimeNs = 0; + long modelDecodeTimeNs = 0; + + for (int j = 0; j < timeOfRepeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledIntsAll, i, scaledInts, 0, end - i); + + long[] values = intsToLongsZigZag(scaledInts); + + long startTime = System.nanoTime(); + long[] encoded = simple8bEncodeAll(values); + long duration = System.nanoTime() - startTime; + modelTimeNs += duration; + modelCostBits += encoded.length * 64L; + + long startDecodeTime = System.nanoTime(); + long[] decodedValues = simple8bDecodeAll(encoded, values.length); + int[] decoded = longsToIntsZigZag(decodedValues); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTimeNs += decodeDuration; + + // prevent accidental dead-code elimination + if (decoded.length != scaledInts.length) { + throw new AssertionError("Decoded length mismatch"); + } + } + } + + modelCostBits /= timeOfRepeat; + modelTimeNs /= timeOfRepeat; + modelDecodeTimeNs /= timeOfRepeat; + + double ratio = (double) modelCostBits / (double) (numbers.size() * 64L); + double encThroughput = (double) (numbers.size() * 8000L) / (double) modelTimeNs; + double decThroughput = (double) (numbers.size() * 8000L) / (double) modelDecodeTimeNs; + + String[] record = { + file.toString(), + "Simple8b", + String.valueOf(encThroughput), + String.valueOf(decThroughput), + String.valueOf(numbers.size()), + String.valueOf(modelCostBits), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Encoding throughput: " + encThroughput + " MB/s"); + System.out.println("Decoding throughput: " + decThroughput + " MB/s"); + System.out.println("Compression ratio: " + ratio); + } + } + + /** + * FastLanes compression test (https://github.com/cwida/FastLanes). + * Invokes C++ executable via ProcessBuilder; skip if executable is not available. + */ + @Test + public void FastLanesTest() throws IOException { + System.out.println("\nPerformance Testing (FastLanes)..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_FastLanes"; + + File dir = new File(directory); + Assume.assumeTrue( + "Skip FastLanesTest: dataset directory missing: " + directory, + dir.exists() && dir.isDirectory() + ); + + File outputDir = new File(outputDirstr); + if (!outputDir.exists()) outputDir.mkdir(); + + String fastLanesCppBin = getFastLanesCppCommand(); + Assume.assumeTrue( + "Skip FastLanesTest: C++ FastLanes executable not found: " + fastLanesCppBin, + fastLanesCppBin != null && !fastLanesCppBin.isEmpty() && new File(fastLanesCppBin).exists() + ); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + + String output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Throughput (MB/s)", + "Decoding Throughput (MB/s)", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) numbers.add(numStr); + } + } + + File runInputDir = Files.createTempDirectory("fastlanes_cpp_in_").toFile(); + File runOutputDir = Files.createTempDirectory("fastlanes_cpp_out_").toFile(); + try { + // Run C++ program on a temp dir containing only the current CSV. + File inputCsv = new File(runInputDir, file.getName()); + Files.copy(file.toPath(), inputCsv.toPath()); + + double[] result = runFastLanesCppOnSingleFile( + fastLanesCppBin, + runInputDir.getAbsolutePath(), + runOutputDir.getAbsolutePath(), + file.getName() + ); + if (result == null) { + String msg = lastFastLanesError != null && !lastFastLanesError.isEmpty() + ? ("Skip FastLanesTest: " + lastFastLanesError.replaceAll("\\s+", " ").trim()) + : "Skip FastLanesTest: C++ FastLanes run failed"; + Assume.assumeTrue(msg, false); + return; + } + double encThroughput = result[0]; + double decThroughput = result[1]; + long modelCostBits = (long) result[2]; + + double ratio = (double) modelCostBits / (double) (numbers.size() * 64L); + + String[] record = { + file.toString(), + "FastLanes", + String.valueOf(encThroughput), + String.valueOf(decThroughput), + String.valueOf(numbers.size()), + String.valueOf(modelCostBits), + String.valueOf(ratio) + }; + writer.writeRecord(record); + + System.out.println("Encoding throughput: " + encThroughput + " MB/s"); + System.out.println("Decoding throughput: " + decThroughput + " MB/s"); + System.out.println("Compression ratio: " + ratio); + } finally { + deleteRecursively(runInputDir); + deleteRecursively(runOutputDir); + } + writer.close(); + } + } + + /** Resolve C++ FastLanes executable. Use -Dfastlanes.cpp.bin=/path/to/optimal_pack_prune to override. */ + private static String getFastLanesCppCommand() { + String override = System.getProperty("fastlanes.cpp.bin"); + if (override != null && !override.isEmpty()) { + File f = new File(override); + if (f.exists() && f.canExecute()) return f.getAbsolutePath(); + } + + // Common locations in this workspace. + String[] candidates = new String[]{ + "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/optimal_pack_prune", + "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/cpp_simd/build/optimal_pack_prune" + }; + for (String candidate : candidates) { + File f = new File(candidate); + if (f.exists() && f.canExecute()) return f.getAbsolutePath(); + } + return ""; + } + + private static String lastFastLanesError; + + /** Returns [ENC_THROUGHPUT_MBPS, DEC_THROUGHPUT_MBPS, COMPRESSED_BITS] from BP+Prune-FastLanes row. */ + private static double[] runFastLanesCppOnSingleFile(String cppBin, String inputDir, String outputDir, String datasetFileName) { + lastFastLanesError = null; + try { + ProcessBuilder pb = new ProcessBuilder( + cppBin, + inputDir, + outputDir + ); + pb.redirectErrorStream(true); + Process p = pb.start(); + StringBuilder output = new StringBuilder(); + try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + output.append(line).append('\n'); + } + } + int exitCode = p.waitFor(); + if (exitCode != 0) { + lastFastLanesError = output.length() > 0 ? output.toString().trim() : "exitCode=" + exitCode; + return null; + } + + File outCsv = new File(outputDir, datasetFileName); + if (!outCsv.exists()) { + lastFastLanesError = "C++ FastLanes output CSV not found: " + outCsv.getAbsolutePath(); + return null; + } + + CsvReader reader = new CsvReader(outCsv.getAbsolutePath(), ',', StandardCharsets.UTF_8); + while (reader.readRecord()) { + String[] vals = reader.getValues(); + if (vals.length < 7) { + continue; + } + String algo = vals[1].replace("\"", "").trim(); + if (!"BP+Prune-FastLanes".equals(algo)) { + continue; + } + double encThroughput = Double.parseDouble(vals[2].replace("\"", "").trim()); + double decThroughput = Double.parseDouble(vals[3].replace("\"", "").trim()); + double compressedBits = Double.parseDouble(vals[5].replace("\"", "").trim()); + return new double[]{ + encThroughput, + decThroughput, + compressedBits + }; + } + + lastFastLanesError = "BP+Prune-FastLanes row not found in " + outCsv.getAbsolutePath(); + return null; + } catch (Exception e) { + lastFastLanesError = e.getMessage(); + return null; + } + } + + private static void deleteRecursively(File f) { + if (f.isDirectory()) { + File[] children = f.listFiles(); + if (children != null) for (File c : children) deleteRecursively(c); + } + f.delete(); + } + + @Test + public void Simple8bRoundTripRandomIntTest() { + Random r = new Random(20260302L); + int n = 50_000; + int[] src = new int[n]; + for (int i = 0; i < n; i++) { + int v; + int p = r.nextInt(100); + if (p < 40) v = 0; + else if (p < 70) v = r.nextInt(1 << 7); + else if (p < 90) v = r.nextInt(1 << 20); + else v = r.nextInt(1 << 28); + src[i] = v; + } + int[] dst = simple8bRoundTripInts(src); + if (!Arrays.equals(src, dst)) { + throw new AssertionError("Simple8b random round-trip mismatch"); + } + } + + private static int[] simple8bRoundTripInts(int[] values) { + long[] src = intsToLongsZigZag(values); + long[] encoded = simple8bEncodeAll(src); + long[] decoded = simple8bDecodeAll(encoded, src.length); + return longsToIntsZigZag(decoded); + } + + // ---------------- Simple8b (Anh & Moffat 2010) ---------------- + // selector: 0..15 in top 4 MSBs, payload: remaining 60 bits + private static final int[] SIMPLE8B_BITS = + {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + private static final int[] SIMPLE8B_N = + {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + private static final long[] SIMPLE8B_MASK = buildSimple8bMasks(); + + private static long[] buildSimple8bMasks() { + long[] m = new long[16]; + for (int i = 0; i < 16; i++) { + int b = SIMPLE8B_BITS[i]; + m[i] = (b == 0 || b == 64) ? 0 : ((1L << b) - 1L); + } + return m; + } + + private static long[] intsToLongsZigZag(int[] src) { + long[] out = new long[src.length]; + for (int i = 0; i < src.length; i++) { + int v = src[i]; + out[i] = (((long) v) << 1) ^ (v >> 31); + } + return out; + } + + private static int[] longsToIntsZigZag(long[] src) { + int[] out = new int[src.length]; + for (int i = 0; i < src.length; i++) { + long v = src[i]; + out[i] = (int) ((v >>> 1) ^ -(v & 1L)); + } + return out; + } + + private static long[] simple8bEncodeAll(long[] values) { + int cap = Math.max(128, (values.length + 239) / 240); + long[] out = new long[cap]; + int outLen = 0; + int i = 0; + while (i < values.length) { + if (outLen == out.length) { + out = Arrays.copyOf(out, out.length * 2); + } + int selector = simple8bSelect(values, i, values.length - i); + int n = SIMPLE8B_N[selector]; + out[outLen++] = simple8bPack(values, i, selector); + i += n; + } + return Arrays.copyOf(out, outLen); + } + + private static long[] simple8bDecodeAll(long[] words, int valueCount) { + long[] out = new long[valueCount]; + int outPos = 0; + for (long word : words) { + int selector = (int) (word >>> 60); + int n = SIMPLE8B_N[selector]; + if (outPos + n > valueCount) { + throw new IllegalArgumentException("Decoded past expected valueCount"); + } + outPos += simple8bUnpack(word, out, outPos); + } + if (outPos != valueCount) { + throw new IllegalArgumentException("Decoded valueCount mismatch: expected=" + valueCount + ", actual=" + outPos); + } + return out; + } + + private static int simple8bSelect(long[] src, int off, int remaining) { + int scan = Math.min(remaining, 240); + long max = 0; + boolean allZero = true; + for (int i = 0; i < scan; i++) { + long v = src[off + i]; + if (v != 0) { + allZero = false; + if (v > max) max = v; + } + } + int neededBits = (max == 0) ? 0 : (64 - Long.numberOfLeadingZeros(max)); + for (int selector = 0; selector < 16; selector++) { + int n = SIMPLE8B_N[selector]; + if (n > remaining) continue; + int bits = SIMPLE8B_BITS[selector]; + if (bits == 0) { + if (allZero) return selector; + continue; + } + if (neededBits <= bits) return selector; + } + return 15; + } + + private static long simple8bPack(long[] src, int off, int selector) { + int bits = SIMPLE8B_BITS[selector]; + int n = SIMPLE8B_N[selector]; + long word = ((long) selector) << 60; + if (bits == 0) return word; + + long mask = SIMPLE8B_MASK[selector]; + for (int i = 0; i < n; i++) { + int shift = 60 - bits * (i + 1); + word |= ((src[off + i] & mask) << shift); + } + return word; + } + + private static int simple8bUnpack(long word, long[] dst, int off) { + int selector = (int) (word >>> 60); + int bits = SIMPLE8B_BITS[selector]; + int n = SIMPLE8B_N[selector]; + if (bits == 0) { + Arrays.fill(dst, off, off + n, 0L); + return n; + } + long mask = SIMPLE8B_MASK[selector]; + for (int i = 0; i < n; i++) { + int shift = 60 - bits * (i + 1); + dst[off + i] = (word >>> shift) & mask; + } + return n; + } + + /// vary pack size + + @Test + public void OptimalPackSizeN2Test() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_N2_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 500; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { int end = Math.min(i + CHUNK_SIZE, numbers.size()); int[] scaledInts = new int[end - i]; if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); @@ -4029,10 +4808,10 @@ public void OptimalPackSizeN2Test() throws IOException { } @Test - public void OptimalPackSizeN2SprintzTest() throws IOException { + public void OptimalPackSizeN2SortTest() throws IOException { System.out.println("\nPerformance Testing..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_N2_all_no8"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_N2_all_no8_sort"; File outputDir = new File(outputDirstr); if (!outputDir.exists()) outputDir.mkdir(); @@ -4107,13 +4886,13 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { int totalCost = 0; for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); + quickSortDesc(scaledInts,0,scaledInts.length - 1); // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV2(scaledInts); + int pack_size = findOptimalPackSizeallForSort(scaledInts); // 确保pack_size至少为1 pack_size = Math.max(1, pack_size); @@ -4144,7 +4923,6 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { // 测试解压性能 long startDecodeTime = System.nanoTime(); int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); long decodeDuration = System.nanoTime() - startDecodeTime; modelDecodeTime += decodeDuration; @@ -4161,7 +4939,7 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { String[] record = { file.toString(), - "Sprintz+RMQ", + "BP+RMQ", String.valueOf(modelTime_throughput), String.valueOf(modelDecodeTime_throughput), String.valueOf(numbers.size()), @@ -4177,17 +4955,19 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { } } + + @Test - public void OptimalPackSizeRMQTest() throws IOException { + public void OptimalPackSizeN2SprintzTest() throws IOException { System.out.println("\nPerformance Testing..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_N2_all_no8"; File outputDir = new File(outputDirstr); if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); for (File file : Objects.requireNonNull(dir.listFiles())) { - if(!file.getName().equals("PM10-dust.csv")) continue; + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; System.out.println(file.getName()); String Output = outputDirstr + "/" + file.getName(); @@ -4224,7 +5004,7 @@ public void OptimalPackSizeRMQTest() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = 50; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -4256,12 +5036,13 @@ public void OptimalPackSizeRMQTest() throws IOException { int totalCost = 0; for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3(scaledInts); + int pack_size = findOptimalPackSizeallV2(scaledInts); // 确保pack_size至少为1 pack_size = Math.max(1, pack_size); @@ -4292,6 +5073,7 @@ public void OptimalPackSizeRMQTest() throws IOException { // 测试解压性能 long startDecodeTime = System.nanoTime(); int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); long decodeDuration = System.nanoTime() - startDecodeTime; modelDecodeTime += decodeDuration; @@ -4308,7 +5090,7 @@ public void OptimalPackSizeRMQTest() throws IOException { String[] record = { file.toString(), - "BP+RMQ", + "Sprintz+RMQ", String.valueOf(modelTime_throughput), String.valueOf(modelDecodeTime_throughput), String.valueOf(numbers.size()), @@ -4325,10 +5107,10 @@ public void OptimalPackSizeRMQTest() throws IOException { } @Test - public void OptimalPackSizeRMQSprintzTest() throws IOException { + public void OptimalPackSizeN2SprintzSortTest() throws IOException { System.out.println("\nPerformance Testing..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8_sprintz"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_N2_all_no8_sort"; File outputDir = new File(outputDirstr); if (!outputDir.exists()) outputDir.mkdir(); @@ -4408,8 +5190,9 @@ public void OptimalPackSizeRMQSprintzTest() throws IOException { long startTime = System.nanoTime(); int[] scaledInts = sprintz(scaledInt); + quickSortDesc(scaledInts,0,scaledInts.length - 1); // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3(scaledInts); + int pack_size = findOptimalPackSizeallForSort(scaledInts); // 确保pack_size至少为1 pack_size = Math.max(1, pack_size); @@ -4427,56 +5210,1206 @@ public void OptimalPackSizeRMQSprintzTest() throws IOException { } } - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void OptimalPackSizeRMQTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + if(!file.getName().equals("PM10-dust.csv")) continue; + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void OptimalPackSizeRMQSortTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8_sort"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + // if(!file.getName().equals("PM10-dust.csv")) continue; + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + if (!NO_TIME_SERIES_FILES.contains(file.getName())) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 500; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + long startTime = System.nanoTime(); + quickSortDesc(scaledInts,0,scaledInts.length - 1); + // 快速排序排 scaledInts + // Arrays.sort(scaledInts); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3ForSort(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + + @Test + public void OptimalPackSizeRMQSprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8_sprintz"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + /** + * Compare TsFile space, write time, read time: pack size 8 vs optimal pack size. + * Writes each dataset to TsFile with Sprintz encoding and measures metrics. + */ + @Test + public void TsFilePackSize8VsOptimalComparisonTest() throws IOException, WriteProcessException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_tsfile_packsize_comparison"; + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdirs(); + + File dir = new File(directory); + Assume.assumeTrue("Data directory not found: " + directory, dir.exists() && dir.isDirectory()); + + String csvPath = outputDirStr + "/tsfile_comparison.csv"; + CsvWriter writer = new CsvWriter(csvPath, ',', StandardCharsets.UTF_8); + String[] head = { + "Dataset", "Mode", "TsFile Size (bytes)", "Write Time (ns)", "Read Time (ns)", + "Points", "Compression Ratio" + }; + writer.writeRecord(head); + + int warmupRepeats = 2; + int measureRepeats = 5; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = numStr.contains(".") ? numStr.split("\\.")[1].length() : 0; + decimalPlaces.add(decimal); + } + } + } + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += 1024) { + int end = Math.min(numbers.size(), i + 1024); + batches.add(scaleNumbers(numbers.subList(i, end), decimalMax)); + } + int[] scaledInts_all = new int[numbers.size()]; + int idx = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, idx, batch.length); + idx += batch.length; + } + long[] dataAsLong = new long[scaledInts_all.length]; + for (int i = 0; i < scaledInts_all.length; i++) { + dataAsLong[i] = scaledInts_all[i]; + } + + IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create("d1"); + Path path = new Path(deviceID, "sensor_1", true); + List pathList = Collections.singletonList(path); + + // 1. Pack size 8 + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + File tsfile8 = new File(outputDirStr + "/" + file.getName().replace(".csv", "_pack8.tsfile")); + if (tsfile8.exists()) tsfile8.delete(); + + long writeTime8 = 0; + long readTime8 = 0; + for (int r = 0; r < warmupRepeats + measureRepeats; r++) { + if (tsfile8.exists()) tsfile8.delete(); + long t0 = System.nanoTime(); + try (TsFileWriter w = new TsFileWriter(tsfile8)) { + w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); + for (int i = 0; i < dataAsLong.length; i++) { + TSRecord rec = new TSRecord(deviceID, i); + rec.addTuple(new LongDataPoint("sensor_1", dataAsLong[i])); + w.writeRecord(rec); + } + } + long t1 = System.nanoTime(); + if (r >= warmupRepeats) writeTime8 += (t1 - t0); + + long t2 = System.nanoTime(); + try (TsFileReader reader = new TsFileReader(tsfile8)) { + QueryDataSet ds = reader.query(QueryExpression.create(pathList, null)); + while (ds.hasNext()) ds.next(); + } + long t3 = System.nanoTime(); + if (r >= warmupRepeats) readTime8 += (t3 - t2); + } + long size8 = tsfile8.length(); + long avgWrite8 = writeTime8 / measureRepeats; + long avgRead8 = readTime8 / measureRepeats; + double ratio8 = (double) size8 / (numbers.size() * 8.0); + + writer.writeRecord(new String[]{ + file.getName(), "PackSize8", + String.valueOf(size8), String.valueOf(avgWrite8), + String.valueOf(avgRead8), String.valueOf(numbers.size()), + String.format("%.4f", ratio8) + }); + + // 2. Optimal pack size + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(true); + File tsfileOpt = new File(outputDirStr + "/" + file.getName().replace(".csv", "_optimal.tsfile")); + if (tsfileOpt.exists()) tsfileOpt.delete(); + + long writeTimeOpt = 0; + long readTimeOpt = 0; + for (int r = 0; r < warmupRepeats + measureRepeats; r++) { + if (tsfileOpt.exists()) tsfileOpt.delete(); + long t0 = System.nanoTime(); + try (TsFileWriter w = new TsFileWriter(tsfileOpt)) { + w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); + for (int i = 0; i < dataAsLong.length; i++) { + TSRecord rec = new TSRecord(deviceID, i); + rec.addTuple(new LongDataPoint("sensor_1", dataAsLong[i])); + w.writeRecord(rec); + } + } + long t1 = System.nanoTime(); + if (r >= warmupRepeats) writeTimeOpt += (t1 - t0); + + long t2 = System.nanoTime(); + try (TsFileReader reader = new TsFileReader(tsfileOpt)) { + QueryDataSet ds = reader.query(QueryExpression.create(pathList, null)); + while (ds.hasNext()) ds.next(); + } + long t3 = System.nanoTime(); + if (r >= warmupRepeats) readTimeOpt += (t3 - t2); + } + long sizeOpt = tsfileOpt.length(); + long avgWriteOpt = writeTimeOpt / measureRepeats; + long avgReadOpt = readTimeOpt / measureRepeats; + double ratioOpt = (double) sizeOpt / (numbers.size() * 8.0); + + writer.writeRecord(new String[]{ + file.getName(), "OptimalPackSize", + String.valueOf(sizeOpt), String.valueOf(avgWriteOpt), + String.valueOf(avgReadOpt), String.valueOf(numbers.size()), + String.format("%.4f", ratioOpt) + }); + + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + + System.out.printf(" %s: Pack8 size=%d bytes, write=%d ns, read=%d ns | Optimal size=%d bytes, write=%d ns, read=%d ns%n", + file.getName(), size8, avgWrite8, avgRead8, + sizeOpt, avgWriteOpt, avgReadOpt); + } + writer.close(); + System.out.println("Results saved to: " + csvPath); + } + + @Test + public void OptimalPackSizeRMQSprintzSortTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_RMQ_all_no8_sprintz_sort"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + if (!NO_TIME_SERIES_FILES.contains(file.getName())) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = 50; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + + for (int j = 0; j < time_of_repeat; j++) { + int totalCost = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + + long startTime = System.nanoTime(); + // // 快速排序排 scaledInt + // Arrays.sort(scaledInt); + int[] scaledInts = sprintz(scaledInt); + quickSortDesc(scaledInts,0,scaledInts.length - 1); + // 使用优化的方法找到最优pack_size（现在可以是任意整数） + int pack_size = findOptimalPackSizeallV3ForSort(scaledInts); + + // 确保pack_size至少为1 + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + + // 计算每个pack的位宽 + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + long cur_cost = compressedData.length * 8L; // 转换为bit数 + long duration = System.nanoTime() - startTime; + modelTime += (duration); + modelCost += cur_cost; + + // 测试解压性能 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + int[] decodedInts = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += decodeDuration; + + } + + } + modelCost = modelCost / time_of_repeat; + modelTime = (modelTime) / time_of_repeat; + modelDecodeTime = (modelDecodeTime) / time_of_repeat; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "Sprintz+RMQ", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test + public void VaryPackSizeTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_pack_size"; + System.out.println("\nTesting Varying Pack Sizes..."); + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdir(); + + File dir = new File(directory); + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("\nTesting file: " + file.getName()); + String outputFile = outputDirStr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputFile, ',', StandardCharsets.UTF_8); + + // 更新表头 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Pack size", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + // 读取数据 + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + int time_of_repeat = 50; + + // 缩放数据 + int batchSize = 1024; + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个pack size + for (int packSize : packSizes) { + System.out.println("Testing pack size: " + packSize); + + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 编码 + long startEncodeTime = System.nanoTime(); + + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + } + } + + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; + + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + + // 写入结果 + String[] record = { + file.toString(), + "BitPacking", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(totalPoints), + String.valueOf(packSize), + String.valueOf(avgCompressedSize), + String.valueOf(compressionRatio) + }; + writer.writeRecord(record); + + System.out.println(" Pack size: " + packSize + + ", Encode throughput: " + String.format("%.2f", encodeThroughput) + " MB/s" + + ", Decode throughput: " + String.format("%.2f", decodeThroughput) + " MB/s" + + ", Compression ratio: " + String.format("%.4f", compressionRatio)); + } + + writer.close(); + System.out.println("Results saved to: " + outputFile); + } + } + + public static void quickSortDesc(int[] arr, int left, int right) { + if (left >= right) return; + + int pivot = partition(arr, left, right); + quickSortDesc(arr, left, pivot - 1); + quickSortDesc(arr, pivot + 1, right); + } + + private static int partition(int[] arr, int left, int right) { + int pivot = arr[right]; // 选择最右边的元素作为基准 + int i = left - 1; // 小于基准的元素的边界 + + for (int j = left; j < right; j++) { + // 改为 > 实现降序（原来升序是 <） + if (arr[j] > pivot) { + i++; + swap(arr, i, j); + } + } + + swap(arr, i + 1, right); + return i + 1; + } + + private static void swap(int[] arr, int i, int j) { + int temp = arr[i]; + arr[i] = arr[j]; + arr[j] = temp; + } + + @Test + public void VaryPackSizeSortTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_pack_size_sort"; + System.out.println("\nTesting Varying Pack Sizes..."); + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdir(); + + File dir = new File(directory); + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + + System.out.println("\nTesting file: " + file.getName()); + String outputFile = outputDirStr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputFile, ',', StandardCharsets.UTF_8); + + // 更新表头 + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Pack size", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + // 读取数据 + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + } + decimalPlaces.add(decimal); + } + } + } + csvReader.close(); + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + int time_of_repeat = 100; + + // 缩放数据 + int batchSize = 1024; + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + + // 测试每个pack size + for (int packSize : packSizes) { + System.out.println("Testing pack size: " + packSize); + + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 编码 + long startEncodeTime = System.nanoTime(); + // 快速排序排 scaledInts + quickSortDesc(scaledInts,0,scaledInts.length - 1); + // System.out.println(Arrays.toString(scaledInts)); + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + // System.out.println(Arrays.toString(bitWidths)); + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + // break; } + } - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); - } + // 写入结果 + String[] record = { + file.toString(), + "BitPacking", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(totalPoints), + String.valueOf(packSize), + String.valueOf(avgCompressedSize), + String.valueOf(compressionRatio) + }; + writer.writeRecord(record); + System.out.println(" Pack size: " + packSize + + ", Encode throughput: " + String.format("%.2f", encodeThroughput) + " MB/s" + + ", Decode throughput: " + String.format("%.2f", decodeThroughput) + " MB/s" + + ", Compression ratio: " + String.format("%.4f", compressionRatio)); + // break; } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; - - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); - double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); - String[] record = { - file.toString(), - "Sprintz+RMQ", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); writer.close(); - - System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); - System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); - System.out.println("Compression ratio: " + model_ratio); + System.out.println("Results saved to: " + outputFile); + // break; } } - @Test - public void VaryPackSizeTest() throws IOException { + public void VaryPackSizeSprintzTest() throws IOException { String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_vary_pack_size"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_pack_size"; System.out.println("\nTesting Varying Pack Sizes..."); File outputDir = new File(outputDirStr); if (!outputDir.exists()) outputDir.mkdir(); @@ -4559,11 +6492,12 @@ public void VaryPackSizeTest() throws IOException { for (int j = 0; j < time_of_repeat; j++) { for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); // 编码 long startEncodeTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); // 计算需要的组数 int numGroups = (scaledInts.length + packSize - 1) / packSize; @@ -4592,6 +6526,7 @@ public void VaryPackSizeTest() throws IOException { // 解码 long startDecodeTime = System.nanoTime(); int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + int[] sprintzDecoded = sprintzDecode(decodedData); long decodeDuration = System.nanoTime() - startDecodeTime; // // 验证解码结果 @@ -4631,7 +6566,7 @@ public void VaryPackSizeTest() throws IOException { // 写入结果 String[] record = { file.toString(), - "BitPacking", + "Sprintz", String.valueOf(encodeThroughput), String.valueOf(decodeThroughput), String.valueOf(totalPoints), @@ -4653,9 +6588,9 @@ public void VaryPackSizeTest() throws IOException { } @Test - public void VaryPackSizeSprintzTest() throws IOException { + public void VaryPackSizeSprintzSortTest() throws IOException { String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_pack_size"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Sprintz_vary_pack_size_sort"; System.out.println("\nTesting Varying Pack Sizes..."); File outputDir = new File(outputDirStr); if (!outputDir.exists()) outputDir.mkdir(); @@ -4743,6 +6678,8 @@ public void VaryPackSizeSprintzTest() throws IOException { // 编码 long startEncodeTime = System.nanoTime(); + // 快速排序scaledInt + Arrays.sort(scaledInt); int[] scaledInts = sprintz(scaledInt); // 计算需要的组数 @@ -4833,6 +6770,7 @@ public void VaryPackSizeSprintzTest() throws IOException { } } + @Test public void OptimalPackSizePruneTest() throws IOException { System.out.println("\nPerformance Testing..."); @@ -5132,6 +7070,305 @@ public void OptimalPackSizePruneSprintzTest() throws IOException { } } + // ---------- RF model (exported from Python) for pack size prediction ---------- + private static double[] rfScalerMean; + private static double[] rfScalerScale; + /** If non-null, only these indices of the 12 chunk features are used (from rf_features.txt). */ + private static int[] rfFeatureIndices; + private static final List rfTreeFeature = new ArrayList<>(); + private static final List rfTreeThreshold = new ArrayList<>(); + private static final List rfTreeLeft = new ArrayList<>(); + private static final List rfTreeRight = new ArrayList<>(); + private static final List rfTreeValue = new ArrayList<>(); + + /** Load RF model and scaler from files produced by export_rf_for_java.py */ + public static void loadRFModel(File scalerFile, File modelFile) throws IOException { + try (java.util.Scanner sc = new java.util.Scanner(scalerFile, StandardCharsets.UTF_8.name())) { + sc.useLocale(Locale.ROOT); + String line1 = sc.nextLine().trim(); + String line2 = sc.nextLine().trim(); + double[] mean = java.util.Arrays.stream(line1.split("\\s+")).mapToDouble(Double::parseDouble).toArray(); + double[] scale = java.util.Arrays.stream(line2.split("\\s+")).mapToDouble(Double::parseDouble).toArray(); + rfScalerMean = mean; + rfScalerScale = scale; + } + rfTreeFeature.clear(); + rfTreeThreshold.clear(); + rfTreeLeft.clear(); + rfTreeRight.clear(); + rfTreeValue.clear(); + try (java.util.Scanner sc = new java.util.Scanner(modelFile, StandardCharsets.UTF_8.name())) { + sc.useLocale(Locale.ROOT); + int nEstimators = sc.nextInt(); + sc.nextLine(); + for (int t = 0; t < nEstimators; t++) { + int nNodes = sc.nextInt(); + sc.nextLine(); + int[] feat = new int[nNodes]; + double[] th = new double[nNodes]; + int[] left = new int[nNodes]; + int[] right = new int[nNodes]; + double[] val = new double[nNodes]; + for (int i = 0; i < nNodes; i++) { + feat[i] = sc.nextInt(); + th[i] = sc.nextDouble(); + left[i] = sc.nextInt(); + right[i] = sc.nextInt(); + val[i] = sc.nextDouble(); + sc.nextLine(); + } + rfTreeFeature.add(feat); + rfTreeThreshold.add(th); + rfTreeLeft.add(left); + rfTreeRight.add(right); + rfTreeValue.add(val); + } + } + File featuresFile = new File(modelFile.getParent(), "rf_features.txt"); + if (featuresFile.canRead()) { + try (java.util.Scanner sc = new java.util.Scanner(featuresFile, StandardCharsets.UTF_8.name())) { + String line = sc.nextLine().trim(); + String[] parts = line.split(","); + rfFeatureIndices = new int[parts.length]; + for (int i = 0; i < parts.length; i++) rfFeatureIndices[i] = Integer.parseInt(parts[i].trim()); + } + } else { + rfFeatureIndices = null; + } + } + + /** + * Bit-packing encode cost in bits for one chunk (same as OptimalPackSizePruneRMQTest). + */ + public static long encodeChunkBits(int[] scaledInts, int packSize) { + int n = scaledInts.length; + int numPacks = (n + packSize - 1) / packSize; + int[] bitWidths = new int[numPacks]; + for (int si = 0; si < n; si += packSize) { + int endIdx = Math.min(si + packSize, n); + int maxInGroup = 0; + for (int sj = si; sj < endIdx; sj++) { + if (scaledInts[sj] > maxInGroup) maxInGroup = scaledInts[sj]; + } + bitWidths[si / packSize] = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + } + byte[] compressed = encodeBitPackingV2(scaledInts, bitWidths, packSize); + return compressed.length * 8L; + } + + /** + * Compression ratio improvement (%) of findOptimalPackSizeallV5 vs fixed pack size 8: + * (cost_pack8 - cost_opt) / cost_pack8 * 100, where cost is encoded bits. + */ + public static double compressionImprovementPctVsPack8(int[] scaledInts) { + long cost8 = encodeChunkBits(scaledInts, 8); + int opt = findOptimalPackSizeallV5(scaledInts); + opt = Math.max(1, opt); + long costOpt = encodeChunkBits(scaledInts, opt); + if (cost8 <= 0) return 0.0; + return (cost8 - costOpt) * 100.0 / cost8; + } + + /** Compute 12 features from a chunk (same order as Python FEATURE_COLS). */ + public static double[] computeChunkFeatures(int[] scaledInts) { + int n = scaledInts.length; + int max = scaledInts[0], min = scaledInts[0]; + for (int j = 1; j < n; j++) { + if (scaledInts[j] > max) max = scaledInts[j]; + if (scaledInts[j] < min) min = scaledInts[j]; + } + int maxBitWidth = max == 0 ? 1 : (32 - Integer.numberOfLeadingZeros(max)); + int minBitWidth = min == 0 ? 1 : (32 - Integer.numberOfLeadingZeros(min)); + int maxMinBitWidthDiff = Math.max(0, maxBitWidth - minBitWidth); + + int[] bitWidths = new int[n]; + for (int j = 0; j < n; j++) { + int v = scaledInts[j]; + bitWidths[j] = (v == 0 ? 1 : (32 - Integer.numberOfLeadingZeros(v))); + } + double bitWidthSum = 0; + for (int j = 0; j < n; j++) bitWidthSum += bitWidths[j]; + double bitWidthMean = n > 0 ? bitWidthSum / n : 0; + double bitWidthVar = 0; + for (int j = 0; j < n; j++) { + double d = bitWidths[j] - bitWidthMean; + bitWidthVar += d * d; + } + bitWidthVar = n > 0 ? bitWidthVar / n : 0; + double bitWidthStd = Math.sqrt(bitWidthVar); + double diffAbsSum = 0; + for (int j = 0; j < n - 1; j++) diffAbsSum += Math.abs(bitWidths[j + 1] - bitWidths[j]); + double bitWidthDiffAbsMean = (n > 1) ? (diffAbsSum / (n - 1)) : 0; + + int[] bitWidthsSorted = bitWidths.clone(); + Arrays.sort(bitWidthsSorted); + double bitWidthMedian = n > 0 ? (n % 2 == 1 ? bitWidthsSorted[n / 2] : (bitWidthsSorted[n / 2 - 1] + bitWidthsSorted[n / 2]) / 2.0) : 0; + int p90Idx = (int) Math.ceil(0.9 * n) - 1; + if (p90Idx < 0) p90Idx = 0; + double bitWidthP90 = n > 0 ? bitWidthsSorted[Math.min(p90Idx, n - 1)] : 0; + + int bitWidthChangeCount = 0; + for (int j = 0; j < n - 1; j++) { + if (bitWidths[j] != bitWidths[j + 1]) bitWidthChangeCount++; + } + int runCount = 0, runLengthSum = 0; + for (int j = 0; j < n; ) { + int r = 1; + while (j + r < n && bitWidths[j + r] == bitWidths[j]) r++; + runCount++; + runLengthSum += r; + j += r; + } + double bitWidthRunMean = runCount > 0 ? (double) runLengthSum / runCount : 0; + int zeroCount = 0; + for (int j = 0; j < n; j++) if (scaledInts[j] == 0) zeroCount++; + double zeroFrac = n > 0 ? (double) zeroCount / n : 0; + + return new double[]{ + maxBitWidth, minBitWidth, maxMinBitWidthDiff, + bitWidthMean, bitWidthVar, bitWidthStd, bitWidthDiffAbsMean, + bitWidthMedian, bitWidthP90, bitWidthChangeCount, bitWidthRunMean, zeroFrac + }; + } + + /** + * RF predicts compression improvement (%) vs pack size 8 (same label as training CSV). + * Returns average of tree outputs (percentage points). + */ + public static double predictImprovementRF(double[] features) { + if (rfScalerMean == null || rfTreeFeature.isEmpty()) throw new IllegalStateException("RF model not loaded"); + int k = rfScalerMean.length; + double[] x = new double[k]; + for (int i = 0; i < k; i++) { + int idx = (rfFeatureIndices != null) ? rfFeatureIndices[i] : i; + x[i] = (features[idx] - rfScalerMean[i]) / (rfScalerScale[i] != 0 ? rfScalerScale[i] : 1.0); + } + double sum = 0; + for (int t = 0; t < rfTreeFeature.size(); t++) { + int node = 0; + int[] f = rfTreeFeature.get(t); + double[] th = rfTreeThreshold.get(t); + int[] left = rfTreeLeft.get(t); + int[] right = rfTreeRight.get(t); + double[] val = rfTreeValue.get(t); + while (f[node] != -2) { + if (x[f[node]] <= th[node]) node = left[node]; + else node = right[node]; + } + sum += val[node]; + } + return sum / rfTreeFeature.size(); + } + + /** + * Same as OptimalPackSizePruneRMQTest but uses RF-predicted improvement: if predicted gain > 0 use + * findOptimalPackSizeallV5, else pack size 8. Output to output_random_tree. + * Requires export_rf_for_java.py (trained on CompressionImprovementPct). + */ + @Test + public void OptimalPackSizeRFPredictTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_random_tree"; + String rfDir = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size"; + File outputDir = new File(outputDirstr); + if (!outputDir.exists()) outputDir.mkdir(); + File scalerFile = new File(rfDir, "rf_scaler.txt"); + File modelFile = new File(rfDir, "rf_model.txt"); + Assume.assumeTrue("RF model not found. Run: python3 export_rf_for_java.py", scalerFile.canRead() && modelFile.canRead()); + loadRFModel(scalerFile, modelFile); + + File dir = new File(directory); + Assume.assumeTrue("Data directory not found: " + directory, dir.exists() && dir.isDirectory()); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String outputPath = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8); + String[] head = { + "Input Direction", "Encoding Algorithm", "Encoding Time", "Decoding Time", + "Points", "Compressed Size", "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = numStr.contains(".") ? numStr.split("\\.")[1].length() : 0; + decimalPlaces.add(decimal); + } + } + } + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + int batchSize = 1024; + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + batches.add(scaleNumbers(numbers.subList(i, end), decimalMax)); + } + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int idx = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, idx, batch.length); + idx += batch.length; + } + + int time_of_repeat = 500; + long modelCost = 0, modelTime = 0, modelDecodeTime = 0; + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + double[] features = computeChunkFeatures(scaledInts); + double predImp = predictImprovementRF(features); + int pack_size = predImp > 0 ? findOptimalPackSizeallV5(scaledInts) : 8; + pack_size = Math.max(1, pack_size); + + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + for (int si = 0; si < scaledInts.length; si += pack_size) { + int endIdx = Math.min(si + pack_size, scaledInts.length); + int maxInGroup = 0; + for (int sj = si; sj < endIdx; sj++) { + if (scaledInts[sj] > maxInGroup) maxInGroup = scaledInts[sj]; + } + bitWidths[si / pack_size] = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + } + long startTime = System.nanoTime(); + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + modelTime += (System.nanoTime() - startTime); + modelCost += compressedData.length * 8L; + long startDecode = System.nanoTime(); + decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); + modelDecodeTime += (System.nanoTime() - startDecode); + } + } + modelCost /= time_of_repeat; + modelTime /= time_of_repeat; + modelDecodeTime /= time_of_repeat; + double model_ratio = (double) modelCost / (numbers.size() * 64.0); + double encThroughput = (double) (numbers.size() * 8000L) / modelTime; + double decThroughput = (double) (numbers.size() * 8000L) / modelDecodeTime; + + String[] record = { + file.toString(), "BP+RF-improvement-gate", + String.valueOf(encThroughput), String.valueOf(decThroughput), + String.valueOf(numbers.size()), String.valueOf(modelCost), String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + System.out.println("RF-predict encoding throughput: " + encThroughput + " MB/s, compression ratio: " + model_ratio); + } + } + @Test public void OptimalPackSizePruneRMQTest() throws IOException { System.out.println("\nPerformance Testing..."); @@ -5431,6 +7668,134 @@ public void OptimalPackSizePruneRMQSprintzTest() throws IOException { } } + + /** + * ALP + optimal pack size (Prune+RMQ): same output format as OptimalPackSizePruneRMQTest. + * Uses findOptimalPackSizeallV5(scaledInts) to compute optimal pack size per chunk; runs ALP and writes + * results to output_alp_prune_rmq. Runs the ALP C++ runner (alp_prune_rmq_runner) if available. + * Set env ALP_PRUNE_RMQ_RUNNER to the path of alp_prune_rmq_runner, or build ALP with -DALP_BUILD_TOOLS=ON + * and use default path. + */ + @Test + public void OptimalPackSizeALPPruneRMQTest() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_alp_prune_rmq"; + String runnerPath = System.getenv("ALP_PRUNE_RMQ_RUNNER"); + if (runnerPath == null || runnerPath.isEmpty()) { + runnerPath = "/Users/xiaojinzhao/Documents/GitHub/ALP/build/tools/alp_prune_rmq_runner"; + } + File runner = new File(runnerPath); + Assume.assumeTrue("ALP runner not found. Build ALP with -DALP_BUILD_TOOLS=ON and set ALP_PRUNE_RMQ_RUNNER or place runner at " + runnerPath, runner.canExecute()); + File dir = new File(directory); + Assume.assumeTrue("Data directory not found: " + directory, dir.exists() && dir.isDirectory()); + + new File(outputDirstr).mkdirs(); + ProcessBuilder pb = new ProcessBuilder(runnerPath, directory, outputDirstr); + pb.redirectErrorStream(true); + Process p = pb.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8)); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + try { + boolean ok = p.waitFor(300, java.util.concurrent.TimeUnit.SECONDS); + Assume.assumeTrue("ALP runner did not finish in time or failed", ok && p.exitValue() == 0); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Assume.assumeNoException(e); + } + System.out.println("ALP+RMQ+Prune results written to " + outputDirstr); + } + + /** + * For each chunk: features (bit-width stats, etc.) and label {@code CompressionImprovementPct} = + * improvement of findOptimalPackSizeallV5 over fixed pack size 8 (encoded bits), in percent. + */ + @Test + public void OptimalPackSizePruneRMQFeatureOutputTest() throws IOException { + System.out.println("\nOptimalPackSizePruneRMQFeatureOutputTest: output features + CompressionImprovementPct vs pack8"); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_feature"; + File outputDir = new File(outputDirstr); + if (!outputDir.exists()) outputDir.mkdir(); + + String outputFeaturePath = outputDirstr + "/output_feature.csv"; + CsvWriter featureWriter = new CsvWriter(outputFeaturePath, ',', StandardCharsets.UTF_8); + String[] featureHead = {"Dataset", "ChunkIndex", "CompressionImprovementPct", "MaxBitWidth", "MinBitWidth", "MaxMinBitWidthDiff", + "BitWidthMean", "BitWidthVar", "BitWidthStd", "BitWidthDiffAbsMean", + "BitWidthMedian", "BitWidthP90", "BitWidthChangeCount", "BitWidthRunMean", "ZeroFrac"}; + featureWriter.writeRecord(featureHead); + + File dir = new File(directory); + Assume.assumeTrue("Data directory not found: " + directory, dir.exists() && dir.isDirectory()); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName() + " for output_feature..."); + + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = numStr.contains(".") ? numStr.split("\\.")[1].length() : 0; + decimalPlaces.add(decimal); + } + } + } + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + List batches = new ArrayList<>(); + for (int i = 0; i < numbers.size(); i += 1024) { + int end = Math.min(numbers.size(), i + 1024); + batches.add(scaleNumbers(numbers.subList(i, end), decimalMax)); + } + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + int idx = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, idx, batch.length); + idx += batch.length; + } + + int chunkIndex = 0; + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + double improvementPct = compressionImprovementPctVsPack8(scaledInts); + double[] fvec = computeChunkFeatures(scaledInts); + + featureWriter.writeRecord(new String[]{ + file.getName(), + String.valueOf(chunkIndex), + String.valueOf(improvementPct), + String.valueOf(fvec[0]), + String.valueOf(fvec[1]), + String.valueOf(fvec[2]), + String.valueOf(fvec[3]), + String.valueOf(fvec[4]), + String.valueOf(fvec[5]), + String.valueOf(fvec[6]), + String.valueOf(fvec[7]), + String.valueOf(fvec[8]), + String.valueOf((int) fvec[9]), + String.valueOf(fvec[10]), + String.valueOf(fvec[11]) + }); + chunkIndex++; + } + } + featureWriter.close(); + System.out.println("Results saved to " + outputFeaturePath); + } + + @Test public void OptimalPackSizePrunePlusTest() throws IOException { System.out.println("\nPerformance Testing..."); @@ -5479,7 +7844,7 @@ public void OptimalPackSizePrunePlusTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = 500; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java new file mode 100644 index 000000000..c9c57bc72 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java @@ -0,0 +1,576 @@ +package org.apache.tsfile.encoding; // 根据你项目包名调整 + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Assume; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.zip.DataFormatException; + +/** + * Simplified CPU reimplementation of a cuSZp-like plain 1D compressor for testing. + * + * Workflow: + * - read a 1D CSV of numbers (double) + * - for each CHUNK, compute delta predictor (prev value) + * - quantize with absolute error bound eb: q = round((v - pred) / eb) + * - zigzag-encode signed q -> unsigned + * - group into packs (packSize) and choose minimal bitwidth per pack + * - bit-pack into byte[] (store per-block header: packSize, numPacks, bitwidths...) + * - decode to verify correctness and measure times + * + * This is a practical testing harness, not a byte-for-byte match to cuSZp GPU impl. + */ +public class CuSZpCpuTest { + + private static final Set IGNORE_FILES = Collections.emptySet(); + private static final int CHUNK_SIZE = 8192; // per-iteration chunk size; tune as needed + + @Test + public void cuSZpCpu1DTest() throws Exception { + System.out.println("\nCPU cuSZp-like plain-mode Performance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_cuszp_cpu"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + Assume.assumeTrue("Skip test: dataset directory missing: " + directory, dir.exists() && dir.isDirectory()); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing " + file.getName()); + + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size (bits)", + "Compression Ratio" + }; + writer.writeRecord(head); + + // read numbers from CSV into a single double[] (assumes whitespace/commas) + List numbers = readCsvToDoubleList(file); + + if (numbers.isEmpty()) { + System.out.println("Empty file: " + file.getName()); + writer.close(); + continue; + } + + // parameters + double eb = 1e-4; // absolute error bound, 可让用户传参 / 读取 + int packSize = 8; // how many values per pack when computing bitwidth + int repeats = 20; // average times + + long totalEncodeNs = 0; + long totalDecodeNs = 0; + long totalCompressedBits = 0; + + for (int r = 0; r < repeats; r++) { + int index = 0; + while (index < numbers.size()) { + int end = Math.min(index + CHUNK_SIZE, numbers.size()); + double[] chunk = new double[end - index]; + for (int i = 0; i < chunk.length; i++) chunk[i] = numbers.get(index + i); + + long sEnc = System.nanoTime(); + byte[] cmp = encodePlain1D(chunk, eb, packSize); + long eEnc = System.nanoTime(); + + long sDec = System.nanoTime(); + double[] dec = decodePlain1D(cmp, chunk.length, eb, packSize); + long eDec = System.nanoTime(); + + // // optional correctness check (can comment out for speed) + // for (int i = 0; i < chunk.length; i++) { + // double err = Math.abs(chunk[i] - dec[i]); + // if (err > eb + 1e-12) { + // throw new AssertionError("Reconstruction error " + err + " > eb for index " + (index + i)); + // } + // } + + totalEncodeNs += (eEnc - sEnc); + totalDecodeNs += (eDec - sDec); + totalCompressedBits += (long) cmp.length * 8L; + + index = end; + } + } + + double avgEncodeNs = (double) totalEncodeNs / repeats; + double avgDecodeNs = (double) totalDecodeNs / repeats; + double points = numbers.size(); + long avgCompressedBits = totalCompressedBits / repeats; + double ratio = avgCompressedBits / (points * 64.0); // compressed bits / raw bits (64 per double) + // throughput MB/s: points * 8 bytes / time (ns) => (points*8)/avgNs * 1e9 bytes/sec -> /1e6 => MB/s + double encodeThroughput = (points * 8.0) / (avgEncodeNs) * 1e3; // MB/s (since ns -> sec & bytes->MB) + double decodeThroughput = (points * 8.0) / (avgDecodeNs) * 1e3; + + String[] record = { + file.toString(), + "cuSZp-cpu-plain-simplified", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf((long) points), + String.valueOf(avgCompressedBits), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.printf("File=%s encode_throughput=%.2f MB/s decode_throughput=%.2f MB/s ratio=%.4f%n", + file.getName(), encodeThroughput, decodeThroughput, ratio); + } + } + + @Test + public void cuSZpCpu1DOptimalV5Test() throws Exception { + System.out.println("\nCPU cuSZp-like plain-mode (optimal pack V5 per chunk)..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_cuszp_cpu_optimal_v5"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + Assume.assumeTrue("Skip test: dataset directory missing: " + directory, dir.exists() && dir.isDirectory()); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println("Processing (V5 pack) " + file.getName()); + + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size (bits)", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = readCsvToDoubleList(file); + + if (numbers.isEmpty()) { + System.out.println("Empty file: " + file.getName()); + writer.close(); + continue; + } + + double eb = 1e-4; + int repeats = 20; + + long totalEncodeNs = 0; + long totalDecodeNs = 0; + long totalCompressedBits = 0; + + for (int r = 0; r < repeats; r++) { + int index = 0; + while (index < numbers.size()) { + int end = Math.min(index + CHUNK_SIZE, numbers.size()); + double[] chunk = new double[end - index]; + for (int i = 0; i < chunk.length; i++) chunk[i] = numbers.get(index + i); + + long sEnc = System.nanoTime(); + byte[] cmp = encodePlain1DOptimalPackV5(chunk, eb); + long eEnc = System.nanoTime(); + + long sDec = System.nanoTime(); + double[] dec = decodePlain1D(cmp, chunk.length, eb, -1); + long eDec = System.nanoTime(); + + totalEncodeNs += (eEnc - sEnc); + totalDecodeNs += (eDec - sDec); + totalCompressedBits += (long) cmp.length * 8L; + + index = end; + } + } + + double avgEncodeNs = (double) totalEncodeNs / repeats; + double avgDecodeNs = (double) totalDecodeNs / repeats; + double points = numbers.size(); + long avgCompressedBits = totalCompressedBits / repeats; + double ratio = avgCompressedBits / (points * 64.0); + double encodeThroughput = (points * 8.0) / (avgEncodeNs) * 1e3; + double decodeThroughput = (points * 8.0) / (avgDecodeNs) * 1e3; + + String[] record = { + file.toString(), + "cuSZp-cpu+V5pack", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf((long) points), + String.valueOf(avgCompressedBits), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.printf("File=%s encode_throughput=%.2f MB/s decode_throughput=%.2f MB/s ratio=%.4f%n", + file.getName(), encodeThroughput, decodeThroughput, ratio); + } + } + + /** Brute-force argmin_p of the same surrogate as {@link AllNo8PacksizeOptimal#findOptimalPackSizeCuSZpMeta8Bits(long[])}. */ + static int bruteOptimalPackCuSZpMeta8Bits(long[] u, int n) { + if (n < 8) { + return n; + } + int[] bitWidths = new int[n]; + for (int i = 0; i < n; i++) { + bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1L, u[i])); + } + long bestCost = Long.MAX_VALUE; + int bestP = 1; + for (int p = 1; p <= n; p++) { + int m = (n + p - 1) / p; + long c = 8L * m; + for (int i = 0; i < m - 1; i++) { + int start = i * p; + int end = start + p - 1; + int mb = 0; + for (int j = start; j <= end; j++) { + mb = Math.max(mb, bitWidths[j]); + } + c += (long) p * mb; + } + if (m > 0) { + int lastStart = (m - 1) * p; + int mb = 0; + for (int j = lastStart; j < n; j++) { + mb = Math.max(mb, bitWidths[j]); + } + c += (long) (n - lastStart) * mb; + } + if (c < bestCost) { + bestCost = c; + bestP = p; + } + } + return bestP; + } + + @Test + public void testCuSZpMeta8BitsPrunedMatchesBrute() { + Random rnd = new Random(7); + for (int t = 0; t < 80; t++) { + int n = 8 + rnd.nextInt(120); + long[] u = new long[n]; + for (int i = 0; i < n; i++) { + u[i] = rnd.nextInt(2000000); + } + int pruned = AllNo8PacksizeOptimal.findOptimalPackSizeCuSZpMeta8Bits(u); + int brute = bruteOptimalPackCuSZpMeta8Bits(u, n); + assertEquals("n=" + n + " trial=" + t, brute, pruned); + } + } + + @Test + public void testEncodedByteLengthPlain1DLayoutMatchesEncode() throws Exception { + Random rnd = new Random(42); + double eb = 1e-3; + for (int trial = 0; trial < 30; trial++) { + int n = 1 + rnd.nextInt(180); + double[] v = new double[n]; + for (int i = 0; i < n; i++) { + v[i] = rnd.nextGaussian() * 10.0; + } + long[] q = new long[n]; + for (int i = 0; i < n; i++) { + double pred = (i == 0) ? 0.0 : v[i - 1]; + q[i] = Math.round((v[i] - pred) / eb); + } + long[] u = new long[n]; + for (int i = 0; i < n; i++) { + u[i] = zigzagEncode(q[i]); + } + for (int p = 1; p <= n; p++) { + long calc = encodedByteLengthPlain1DLayout(u, n, p); + byte[] enc = encodePlain1D(v, eb, p); + assertEquals("n=" + n + " p=" + p, enc.length, calc); + } + } + } + + // ---------- Encoding / Decoding primitives (simplified plain-mode) ---------- + + /** + * Output size in bytes for {@link #encodePlain1D(double[], double, int)} given precomputed zigzag-unsigned + * {@code u} (same definition as inside {@code encodePlain1D}) and {@code packSize}. Matches the actual layout: + * {@code 4+4+4} byte header, {@code numPacks} width bytes, then bit payload with {@link BitWriter} rules and + * {@code flush()} (partial final byte padded to a full byte). + */ + static long encodedByteLengthPlain1DLayout(long[] u, int n, int packSize) { + if (n <= 0) { + return 12L; + } + int numPacks = (n + packSize - 1) / packSize; + long dataBits = 0; + for (int pk = 0; pk < numPacks; pk++) { + int start = pk * packSize; + int end = Math.min(n, start + packSize); + long maxv = 0; + for (int i = start; i < end; i++) { + if (u[i] > maxv) { + maxv = u[i]; + } + } + int bw = (maxv == 0) ? 0 : (64 - Long.numberOfLeadingZeros(maxv)); + dataBits += (long) (end - start) * bw; + } + long payloadBytes = (dataBits + 7) / 8; + return 12L + numPacks + payloadBytes; + } + + /** + * Pack size {@code p} in {@code 1..n} that minimizes {@link #encodedByteLengthPlain1DLayout(long[], int, int)} + * for zigzag-unsigned array {@code u} (length {@code n}). Ties favor smaller {@code p}. + */ + public static int findOptimalPackSizePlain1DExact(long[] u, int n) { + if (n <= 0) { + return 8; + } + long bestBytes = Long.MAX_VALUE; + int bestP = 1; + for (int p = 1; p <= n; p++) { + long len = encodedByteLengthPlain1DLayout(u, n, p); + if (len < bestBytes || (len == bestBytes && p < bestP)) { + bestBytes = len; + bestP = p; + } + } + return bestP; + } + + /** + * Same as {@link #encodePlain1D(double[], double, int)} but chooses {@code packSize} with the same RMQ + prune + * strategy as {@link AllNo8PacksizeOptimal#findOptimalPackSizeallV5(int[])}, while metadata cost uses {@code 8m} + * bits (one byte per pack) instead of {@code m*z}: {@link AllNo8PacksizeOptimal#findOptimalPackSizeCuSZpMeta8Bits(long[])}. + *

+ * That objective matches V5's linear data-bit model + 8 bits per pack; it is not identical to minimizing + * {@link #encodedByteLengthPlain1DLayout(long[], int, int)} (byte alignment / flush). Use + * {@link #findOptimalPackSizePlain1DExact(long[], int)} for exact byte length. + */ + public static byte[] encodePlain1DOptimalPackV5(double[] values, double eb) throws IOException { + long[] q = new long[values.length]; + for (int i = 0; i < values.length; i++) { + double pred = (i == 0) ? 0.0 : values[i - 1]; + q[i] = Math.round((values[i] - pred) / eb); + } + long[] u = new long[q.length]; + for (int i = 0; i < q.length; i++) { + u[i] = zigzagEncode(q[i]); + } + int packSize = AllNo8PacksizeOptimal.findOptimalPackSizeCuSZpMeta8Bits(u); + return encodePlain1D(values, eb, packSize); + } + + /** + * Encode double[] chunk using simple delta predictor + quantization (abs error eb) + pack-size bitpacking. + * Block format (simplified): + * [int32:packSize][int32:len][int32:numPacks][for each pack: int8 bitWidth][data bytes...] + */ + public static byte[] encodePlain1D(double[] values, double eb, int packSize) throws IOException { + // quantize with delta predictor + long[] q = new long[values.length]; + double prev = 0.0; + for (int i = 0; i < values.length; i++) { + double pred = (i == 0) ? 0.0 : values[i - 1]; // 1D previous-value predictor + double diff = values[i] - pred; + long qi = Math.round(diff / eb); // quantized integer (signed) + q[i] = qi; + } + // zigzag encode to unsigned + long[] u = new long[q.length]; + for (int i = 0; i < q.length; i++) u[i] = zigzagEncode(q[i]); + + // compute per-pack bitwidths + int numPacks = (q.length + packSize - 1) / packSize; + int[] bitWidths = new int[numPacks]; + for (int p = 0; p < numPacks; p++) { + int start = p * packSize; + int end = Math.min(q.length, start + packSize); + long maxv = 0; + for (int i = start; i < end; i++) if (u[i] > maxv) maxv = u[i]; + int bw = (maxv == 0) ? 0 : (64 - Long.numberOfLeadingZeros(maxv)); + bitWidths[p] = bw; + } + + // bit pack into ByteArrayOutputStream with a simple header + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + + dos.writeInt(packSize); + dos.writeInt(values.length); + dos.writeInt(numPacks); + // write bitWidths as bytes + for (int bw : bitWidths) dos.writeByte(bw); + + // now write bitstream per pack + BitWriter bwriter = new BitWriter(baos); + for (int p = 0; p < numPacks; p++) { + int start = p * packSize; + int end = Math.min(q.length, start + packSize); + int bwid = bitWidths[p]; + for (int i = start; i < end; i++) { + if (bwid > 0) bwriter.writeBits(u[i], bwid); + // if bwid==0, value is zero and nothing is written for this item + } + // pad pack to align pack boundary? we simply continue (no per-pack padding) + } + bwriter.flush(); + dos.flush(); + return baos.toByteArray(); + } + + /** + * Decode according to the simple format above. + */ + public static double[] decodePlain1D(byte[] cmp, int originalLen, double eb, int expectedPackSize) throws IOException, DataFormatException { + ByteArrayInputStream bais = new ByteArrayInputStream(cmp); + DataInputStream dis = new DataInputStream(bais); + int packSize = dis.readInt(); + int len = dis.readInt(); + int numPacks = dis.readInt(); + int[] bitWidths = new int[numPacks]; + for (int i = 0; i < numPacks; i++) bitWidths[i] = dis.readByte() & 0xFF; + + // build a BitReader starting from current pos + int headerBytes = 4 + 4 + 4 + numPacks; // packSize + len + numPacks + bitwidths + byte[] bitStream = Arrays.copyOfRange(cmp, headerBytes, cmp.length); + BitReader breader = new BitReader(bitStream); + + long[] u = new long[len]; + for (int p = 0; p < numPacks; p++) { + int start = p * packSize; + int end = Math.min(len, start + packSize); + int bw = bitWidths[p]; + for (int i = start; i < end; i++) { + long val = (bw == 0) ? 0L : breader.readBits(bw); + u[i] = val; + } + } + // zigzag decode and re-apply predictor + double[] out = new double[len]; + long[] q = new long[len]; + for (int i = 0; i < len; i++) q[i] = zigzagDecode(u[i]); + for (int i = 0; i < len; i++) { + double pred = (i == 0) ? 0.0 : out[i - 1]; + out[i] = pred + q[i] * eb; + } + return out; + } + + // ---------- helpers ---------- + + private static List readCsvToDoubleList(File f) throws IOException { + List list = new ArrayList<>(); + CsvReader r = new CsvReader(f.getPath(), ',', StandardCharsets.UTF_8); + while (r.readRecord()) { + for (String v : r.getValues()) { + String s = v.trim(); + if (!s.isEmpty()) { + try { + list.add(Double.parseDouble(s)); + } catch (NumberFormatException ex) { + // skip non-number cells + } + } + } + } + return list; + } + + // ZigZag: map signed -> unsigned (so small negatives become small unsigned) + private static long zigzagEncode(long x) { + return (x << 1) ^ (x >> 63); + } + + private static long zigzagDecode(long u) { + return (u >>> 1) ^ -(u & 1); + } + + // Simple bit writer (append bits, LSB-first within value) + static class BitWriter { + private final ByteArrayOutputStream baos; + private int bitPos = 0; // next bit to write into current byte (0..7) + private int currentByte = 0; + + BitWriter(ByteArrayOutputStream baos) { + this.baos = baos; + } + + // write the low 'bits' bits of value (value treated as unsigned) + void writeBits(long value, int bits) throws IOException { + for (int i = bits - 1; i >= 0; i--) { + int bit = (int) ((value >> i) & 1L); + currentByte = (currentByte << 1) | bit; + bitPos++; + if (bitPos == 8) { + baos.write((byte) currentByte); + bitPos = 0; + currentByte = 0; + } + } + } + + void flush() throws IOException { + if (bitPos > 0) { + // pad the remaining bits in the last byte (left-shift to MSB) + int shiftLeft = 8 - bitPos; + currentByte = currentByte << shiftLeft; + baos.write((byte) currentByte); + bitPos = 0; + currentByte = 0; + } + } + } + + // Simple bit reader (reads bits produced by BitWriter) + static class BitReader { + private final byte[] data; + private int byteIdx = 0; + private int bitIdx = 0; // next bit to read inside current byte (0..7, MSB-first as we wrote) + + BitReader(byte[] data) { + this.data = data; + this.byteIdx = 0; + this.bitIdx = 0; + } + + // read 'bits' bits and return as long (0 if bits==0) + long readBits(int bits) throws DataFormatException { + if (bits == 0) return 0L; + long out = 0L; + for (int i = 0; i < bits; i++) { + if (byteIdx >= data.length) throw new DataFormatException("BitReader overflow"); + int b = data[byteIdx] & 0xFF; + int shift = 7 - bitIdx; + int bit = (b >> shift) & 1; + out = (out << 1) | bit; + bitIdx++; + if (bitIdx == 8) { + bitIdx = 0; + byteIdx++; + } + } + return out; + } + } +} \ No newline at end of file diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java new file mode 100644 index 000000000..efdd5c5f2 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java @@ -0,0 +1,188 @@ +package org.apache.tsfile.encoding; + +import java.util.*; + +public class HBPIndexLong { + + public static final int W = 64; + + public final int k; + + public final int sectionBits; + + public final int sectionsPerWord; + + public final int codesPerSegment; + + public final int n; + + public final int segments; + + public final long[] words; + + public final long lowKOnesRepeat; + public final long delimiterBitRepeat; + public final long addOneEachSection; + + public enum Op { + EQ, NE, LT, LE, GT, GE + } + + public HBPIndexLong(int kBits, long[] codes) { + + this.k = kBits; + this.sectionBits = k + 1; + this.sectionsPerWord = W / sectionBits; + if (sectionsPerWord <= 0) + throw new IllegalArgumentException("k too large for 64-bit word"); + this.codesPerSegment = sectionsPerWord * (k + 1); + this.n = codes.length; + this.segments = (n + codesPerSegment - 1) / codesPerSegment; + this.words = new long[segments * (k + 1)]; + + this.lowKOnesRepeat = repeatInSections((1L << k) - 1L); + this.delimiterBitRepeat = repeatInSections(1L << k); + this.addOneEachSection = repeatInSections(1L); + + pack(codes); + } + + public BitSet select(Op op, long C) { + return selectInternal(op, C & ((1L << k) - 1)); + } + + public long count(Op op, long C) { + BitSet bs = select(op, C); + return bs.cardinality(); + } + + public int size() { + return n; + } + + public long getCode(int index) { + if (index < 0 || index >= n) + throw new IndexOutOfBoundsException(); + int s = index / codesPerSegment; + int posInSeg = index - s * codesPerSegment; + int i = posInSeg % (k + 1); + int j = posInSeg / (k + 1); + long word = words[s * (k + 1) + i]; + long section = (word >>> (j * sectionBits)) & ((1L << sectionBits) - 1L); + return (section & ((1L << k) - 1L)); + } + + public void pack(long[] codes) { + long sectionMask = (1L << sectionBits) - 1L; + for (int s = 0; s < segments; s++) { + int base = s * codesPerSegment; + int upto = Math.min(n, base + codesPerSegment); + int count = upto - base; + for (int i = 0; i <= k; i++) { + long w = 0L; + for (int j = 0; j < sectionsPerWord; j++) { + int idx = base + i + j * (k + 1); + if (idx >= upto) + break; + long code = codes[idx] & ((1L << k) - 1L); + long section = code; + w |= (section & sectionMask) << (j * sectionBits); + } + words[s * (k + 1) + i] = w; + } + } + } + + public BitSet selectInternal(Op op, long Ck) { + BitSet out = new BitSet(n); + long yRepeat = repeatInSections(Ck & ((1L << k) - 1)); + for (int s = 0; s < segments; s++) { + long segBits = 0L; + for (int i = 0; i <= k; i++) { + + long X = words[s * (k + 1) + i]; + long Z = fOp(op, X, yRepeat); + + long shifted = Z >>> (k - i); + segBits |= shifted; + } + + int base = s * codesPerSegment; + int valid = Math.min(codesPerSegment, n - base); + long mask = valid == 64 ? ~0L : ((1L << valid) - 1L); + segBits &= mask; + + int bitIndexBase = s * codesPerSegment; + while (segBits != 0) { + int t = Long.numberOfTrailingZeros(segBits); + out.set(bitIndexBase + t); + segBits &= (segBits - 1); + } + } + return out; + } + + public long fOp(Op op, long X, long Yrepeat) { + + switch (op) { + case NE: + return ((X ^ Yrepeat) + lowKOnesRepeat) & delimiterBitRepeat; + case EQ: + return (~((X ^ Yrepeat) + lowKOnesRepeat)) & delimiterBitRepeat; + case LT: + return (Yrepeat + (X ^ lowKOnesRepeat)) & delimiterBitRepeat; + case LE: + return (Yrepeat + (X ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat; + case GT: + return (X + (Yrepeat ^ lowKOnesRepeat)) & delimiterBitRepeat; + case GE: + return (X + (Yrepeat ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat; + } + + return 0L; + } + + public long repeatInSections(long payload) { + long res = 0L; + for (int j = 0; j < sectionsPerWord; j++) { + res |= (payload & ((1L << sectionBits) - 1L)) << (j * sectionBits); + } + return res; + } + + + public static void main(String[] args) { + int k = 3; + long[] codes = { + 1, 5, 6, 1, 6, 4, 0, 7, 4, 3 + }; + HBPIndexLong idx = new HBPIndexLong(k, codes); + + for (int i = 0; i < idx.words.length; i++) { + System.out.printf("word[%d] = %016X\n", i, idx.words[i]); + } + + System.out.println("segments: " + idx.segments); + + System.out.println("lowKOnesRepeat: " + String.format("%64s", Long.toBinaryString(idx.lowKOnesRepeat)).replace(' ', '0')); + System.out.println("delimiterBitRepeat: " + String.format("%64s", Long.toBinaryString(idx.delimiterBitRepeat)).replace(' ', '0')); + System.out.println("addOneEachSection: " + String.format("%64s", Long.toBinaryString(idx.addOneEachSection)).replace(' ', '0')); + + System.out.println("n = " + idx.size()); + + BitSet lt5 = idx.select(Op.LT, 4); + System.out.println("< 4 -> " + lt5); + + BitSet eq4 = idx.select(Op.EQ, 4); + System.out.println("= 4 -> " + eq4); + + BitSet ge6 = idx.select(Op.GE, 6); + System.out.println(">= 6 -> " + ge6); + + for (int i = 0; i < idx.size(); i++) { + System.out.print(idx.getCode(i) + (i + 1 == idx.size() ? "\n" : " ")); + } + + System.out.println("count(<5) = " + idx.count(Op.LT, 5)); + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java new file mode 100644 index 000000000..d659ec0a1 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java @@ -0,0 +1,311 @@ +package org.apache.tsfile.encoding; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import org.junit.Assume; +import org.junit.Test; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; + +public class HBPIndexLongTest { + + private static final List IGNORE_FILES = Arrays.asList( + ".DS_Store", "full_data", "test.csv", "POI-lat.csv", "POI-lon.csv", + "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv"); + + public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 24); + cur_byte[encode_pos + 1] = (byte) (integer >> 16); + cur_byte[encode_pos + 2] = (byte) (integer >> 8); + cur_byte[encode_pos + 3] = (byte) (integer); + } + + public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) { + cur_byte[encode_pos] = (byte) (integer >> 56); + cur_byte[encode_pos + 1] = (byte) (integer >> 48); + cur_byte[encode_pos + 2] = (byte) (integer >> 40); + cur_byte[encode_pos + 3] = (byte) (integer >> 32); + cur_byte[encode_pos + 4] = (byte) (integer >> 24); + cur_byte[encode_pos + 5] = (byte) (integer >> 16); + cur_byte[encode_pos + 6] = (byte) (integer >> 8); + cur_byte[encode_pos + 7] = (byte) (integer); + } + + public static int bytes2Integer(byte[] encoded, int start, int num) { + int value = 0; + + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + public static long bytes2Long(byte[] encoded, int start, int num) { + long value = 0; + + for (int i = 0; i < num; i++) { + value <<= 8; + int b = encoded[i + start] & 0xFF; + value |= b; + } + return value; + } + + public static int bitWidth(int value) { + return 32 - Integer.numberOfLeadingZeros(value); + } + + public static int bitWidth(long value) { + return 64 - Long.numberOfLeadingZeros(value); + } + + public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder, + int encode_pos, ArrayList indexList, byte[] encoded_result) { + + long[] block_data = new long[remainder]; + System.arraycopy(data, block_index * block_size, block_data, 0, remainder); + + long min_value = Long.MAX_VALUE; + long max_value = Long.MIN_VALUE; + for (long value : block_data) { + if (value < min_value) { + min_value = value; + } + if (value > max_value) { + max_value = value; + } + } + + for (int i = 0; i < remainder; i++) { + block_data[i] -= min_value; + } + + long2Bytes(min_value, encode_pos, encoded_result); + encode_pos += 8; + + int bw = bitWidth(max_value - min_value); + + int2Bytes(bw, encode_pos, encoded_result); + encode_pos += 4; + + HBPIndexLong idx = new HBPIndexLong(bw, block_data); + indexList.add(idx); + + return encode_pos; + + } + + public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder, + int encode_pos, ArrayList indexList, long[] data) { + + long min_value = bytes2Long(encoded_result, encode_pos, 8); + encode_pos += 8; + + int bw = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + + HBPIndexLong idx = indexList.get(block_index); + + for (int i = 0; i < remainder; i++) { + long value = idx.getCode(i); + + data[block_index * block_size + i] = value + min_value; + } + + return encode_pos; + + } + + public static int Encoder(long[] data, int block_size, ArrayList indexList, byte[] encoded_result) { + int data_length = data.length; + int encode_pos = 0; + + int2Bytes(data_length, encode_pos, encoded_result); + encode_pos += 4; + + int2Bytes(block_size, encode_pos, encoded_result); + encode_pos += 4; + + int num_blocks = data_length / block_size; + + int remainder = data_length % block_size; + + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, indexList, encoded_result); + } + + encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, indexList, + encoded_result); + + return encode_pos; + } + + public static long[] Decoder(byte[] encoded_result, ArrayList indexList) { + int encode_pos = 0; + + int data_length = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + + int block_size = bytes2Integer(encoded_result, encode_pos, 4); + encode_pos += 4; + + int num_blocks = data_length / block_size; + + long[] data = new long[data_length]; + + for (int i = 0; i < num_blocks; i++) { + encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, data); + } + + int remainder = data_length % block_size; + + encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, + encode_pos, indexList, data); + + return data; + } + + public static int getDecimalPrecision(String str) { + int decimalIndex = str.indexOf("."); + + if (decimalIndex == -1) { + return 0; + } + + return str.substring(decimalIndex + 1).length(); + } + + public static String extractFileName(String path) { + if (path == null || path.isEmpty()) { + return ""; + } + + File file = new File(path); + String fileName = file.getName(); + + int dotIndex = fileName.lastIndexOf('.'); + + if (dotIndex == -1 || dotIndex == 0) { + return fileName; + } + + return fileName.substring(0, dotIndex); + } + + @Test + public void test0() throws IOException { + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Bitweaving"; + int block_size = 512; + int repeatTime = 50; + + File dir = new File(directory); + Assume.assumeTrue( + "Skip test0: dataset directory missing: " + directory, + dir.exists() && dir.isDirectory() + ); + + File outputDir = new File(outputDirStr); + if (!outputDir.exists()) outputDir.mkdir(); + + for (File file : Objects.requireNonNull(dir.listFiles())) { + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + if (!file.getName().toLowerCase().endsWith(".csv")) continue; + + System.out.println(file.getName()); + String outputPath = outputDirStr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Throughput (MB/s)", + "Decoding Throughput (MB/s)", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + + List numbers = new ArrayList<>(); + int max_decimal = 0; + CsvReader loader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (loader.readRecord()) { + for (String value : loader.getValues()) { + String numStr = value.trim(); + if (numStr.isEmpty()) continue; + numbers.add(numStr); + int cur = getDecimalPrecision(numStr); + if (cur > max_decimal) max_decimal = cur; + } + } + loader.close(); + if (max_decimal > 17) max_decimal = 17; + + ArrayList data1 = new ArrayList<>(); + for (String numStr : numbers) { + data1.add(Double.valueOf(numStr)); + } + + long max_mul = (long) Math.pow(10, max_decimal); + long[] data2_arr = new long[data1.size()]; + for (int i = 0; i < data1.size(); i++) { + data2_arr[i] = (long) (data1.get(i) * max_mul); + } + + byte[] encoded_result = new byte[data2_arr.length * 8]; + ArrayList indexList = new ArrayList<>(); + + long s = System.nanoTime(); + int length = 0; + for (int repeat = 0; repeat < repeatTime; repeat++) { + indexList.clear(); + length = Encoder(data2_arr, block_size, indexList, encoded_result); + } + long e = System.nanoTime(); + long encodeTimeNs = (e - s) / repeatTime; + + long compressedBytes = length; + for (HBPIndexLong idx : indexList) { + compressedBytes += idx.segments * (idx.k + 1) * Long.BYTES; + } + long compressedSizeBits = compressedBytes * 8L; + double ratio = (double) compressedSizeBits / (double) (data1.size() * 64L); + + long[] data2_arr_decoded = new long[data2_arr.length]; + s = System.nanoTime(); + for (int repeat = 0; repeat < repeatTime; repeat++) { + data2_arr_decoded = Decoder(encoded_result, indexList); + } + e = System.nanoTime(); + long decodeTimeNs = (e - s) / repeatTime; + + double encodeThroughput = (double) (data1.size() * 8000L) / (double) encodeTimeNs; + double decodeThroughput = (double) (data1.size() * 8000L) / (double) decodeTimeNs; + + String[] record = { + file.toString(), + "HBP", + String.valueOf(encodeThroughput), + String.valueOf(decodeThroughput), + String.valueOf(data1.size()), + String.valueOf(compressedSizeBits), + String.valueOf(ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Encoding throughput: " + encodeThroughput + " MB/s, Decoding throughput: " + decodeThroughput + " MB/s, Ratio: " + ratio); + } + } + +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/SprintzOptimalPackSizeBenchmarkTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/SprintzOptimalPackSizeBenchmarkTest.java new file mode 100644 index 000000000..c36b959cc --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/SprintzOptimalPackSizeBenchmarkTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encoding.decoder.LongSprintzDecoder; +import org.apache.tsfile.encoding.encoder.LongSprintzEncoder; +import org.apache.tsfile.encoding.optimal.SprintzOptimalPackSize; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * Benchmark test for comparing Sprintz encoding with default block size (8) vs optimal block size. + * Demonstrates how to measure storage space and read/write time before and after optimization. + */ +public class SprintzOptimalPackSizeBenchmarkTest { + + private static final int SAMPLE_SIZE = 10240; + private static final int WARMUP_ITERATIONS = 3; + private static final int MEASURE_ITERATIONS = 10; + + @Before + public void setUp() { + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + } + + /** + * Generate sample residuals (Sprintz transform format) from raw values for optimal pack size + * computation. Matches LongSprintzEncoder's delta predict + transform: pred = value - prev; + * residual = pred <= 0 ? -2*pred : 2*pred-1. + */ + private long[] getResidualsForOptimal(long[] rawValues) { + if (rawValues.length < 2) { + return rawValues; + } + long[] residuals = new long[rawValues.length - 1]; + long prev = rawValues[0]; + for (int i = 1; i < rawValues.length; i++) { + long pred = rawValues[i] - prev; + residuals[i - 1] = pred <= 0 ? -2 * pred : 2 * pred - 1; + prev = rawValues[i]; + } + return residuals; + } + + @Test + public void testSprintzOptimalPackSizeBenchmark() throws IOException { + long[] testData = generateTestData(SAMPLE_SIZE); + + System.out.println("=== Sprintz Long Encoding Benchmark (Optimization Comparison) ===\n"); + System.out.println("Sample size: " + SAMPLE_SIZE + " values\n"); + + // 1. Default block size (8) + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + BenchmarkResult defaultResult = runBenchmark(testData); + System.out.println("--- Default (Block size = 8) ---"); + printResult(defaultResult); + + // 2. Find optimal block size from sample + long[] residuals = getResidualsForOptimal(testData); + int optimalBlockSize = SprintzOptimalPackSize.findOptimalPackSize(residuals); + optimalBlockSize = Math.max(1, Math.min(32, optimalBlockSize)); + System.out.println("\nOptimal block size from SprintzOptimalPackSize: " + optimalBlockSize); + + // 3. Optimal block size + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(optimalBlockSize); + BenchmarkResult optimalResult = runBenchmark(testData); + System.out.println("\n--- Optimal (Block size = " + optimalBlockSize + ") ---"); + printResult(optimalResult); + + // 4. Comparison + System.out.println("\n--- Comparison ---"); + double sizeRatio = + (double) optimalResult.compressedSize / (double) defaultResult.compressedSize; + double encodeRatio = + (double) optimalResult.encodeTimeNs / (double) defaultResult.encodeTimeNs; + double decodeRatio = + (double) optimalResult.decodeTimeNs / (double) defaultResult.decodeTimeNs; + System.out.printf( + "Compressed size: %.2f%% of default (smaller is better)%n", sizeRatio * 100); + System.out.printf("Encode time: %.2f%% of default%n", encodeRatio * 100); + System.out.printf("Decode time: %.2f%% of default%n", decodeRatio * 100); + + // 5. Auto optimal mode (each block finds its own optimal pack size) + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(true); + BenchmarkResult autoOptimalResult = runBenchmark(testData); + System.out.println("\n--- Auto Optimal (per-block optimal pack size) ---"); + printResult(autoOptimalResult); + + // Restore default + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + } + + @Test + public void testOptimalModeEncodeDecode() throws IOException { + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(true); + try { + LongSprintzEncoder encoder = new LongSprintzEncoder(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + long[] data = generateTestData(128); + for (long v : data) { + encoder.encode(v, baos); + } + encoder.flush(baos); + + ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); + LongSprintzDecoder decoder = new LongSprintzDecoder(); + for (long expected : data) { + assertTrue("Expected more data", decoder.hasNext(buffer)); + long actual = decoder.readLong(buffer); + assertEquals("Value mismatch", expected, actual); + } + assertFalse("Should have no more data", decoder.hasNext(buffer)); + } finally { + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + } + } + + private long[] generateTestData(int size) { + Random rand = new Random(42); + long[] data = new long[size]; + long base = 1000000L; + for (int i = 0; i < size; i++) { + data[i] = base + (long) (rand.nextGaussian() * 1000); + } + return data; + } + + private BenchmarkResult runBenchmark(long[] data) throws IOException { + long totalEncodeNs = 0; + long totalDecodeNs = 0; + int compressedSize = 0; + + for (int iter = 0; iter < WARMUP_ITERATIONS + MEASURE_ITERATIONS; iter++) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + LongSprintzEncoder encoder = new LongSprintzEncoder(); + + long encodeStart = System.nanoTime(); + for (long v : data) { + encoder.encode(v, baos); + } + encoder.flush(baos); + long encodeEnd = System.nanoTime(); + + byte[] encoded = baos.toByteArray(); + compressedSize = encoded.length; + + ByteBuffer buffer = ByteBuffer.wrap(encoded); + LongSprintzDecoder decoder = new LongSprintzDecoder(); + List decoded = new ArrayList<>(); + + long decodeStart = System.nanoTime(); + while (decoder.hasNext(buffer)) { + decoded.add(decoder.readLong(buffer)); + } + long decodeEnd = System.nanoTime(); + + if (iter >= WARMUP_ITERATIONS) { + totalEncodeNs += (encodeEnd - encodeStart); + totalDecodeNs += (decodeEnd - decodeStart); + } + } + + return new BenchmarkResult( + compressedSize, + totalEncodeNs / MEASURE_ITERATIONS, + totalDecodeNs / MEASURE_ITERATIONS); + } + + private void printResult(BenchmarkResult r) { + double compressionRatio = (double) r.compressedSize / (SAMPLE_SIZE * 8.0); + double encodeThroughput = (SAMPLE_SIZE * 8.0) / (r.encodeTimeNs / 1e9) / (1024 * 1024); + double decodeThroughput = (SAMPLE_SIZE * 8.0) / (r.decodeTimeNs / 1e9) / (1024 * 1024); + System.out.printf(" Compressed size: %d bytes (ratio: %.4f)%n", r.compressedSize, compressionRatio); + System.out.printf(" Encode time: %.3f ms (throughput: %.2f MB/s)%n", r.encodeTimeNs / 1e6, encodeThroughput); + System.out.printf(" Decode time: %.3f ms (throughput: %.2f MB/s)%n", r.decodeTimeNs / 1e6, decodeThroughput); + } + + private static class BenchmarkResult { + final int compressedSize; + final long encodeTimeNs; + final long decodeTimeNs; + + BenchmarkResult(int compressedSize, long encodeTimeNs, long decodeTimeNs) { + this.compressedSize = compressedSize; + this.encodeTimeNs = encodeTimeNs; + this.decodeTimeNs = decodeTimeNs; + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/CsvReadWriteTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/CsvReadWriteTest.java new file mode 100644 index 000000000..ec87c83e4 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/CsvReadWriteTest.java @@ -0,0 +1,770 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.write; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.read.reader.LocalTsFileInput; +import org.apache.tsfile.read.reader.TsFileInput; +import org.apache.tsfile.utils.NoSyncBufferedOutputStream; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.Schema; +import org.apache.tsfile.write.writer.TsFileOutput; + +import com.csvreader.CsvReader; +import com.csvreader.CsvWriter; +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public class CsvReadWriteTest { + + private static final String PARENT_DIR = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; + private static final String INPUT_PARENT_DIR = PARENT_DIR + "dataset_tsfile/"; + // private static final String INPUT_PARENT_DIR = PARENT_DIR + "dataset_long/"; + private static final String OUTPUT_PARENT_DIR = PARENT_DIR + "result/tsfile_read_write/"; + private static final String TSFILE_OUTPUT_DIR = OUTPUT_PARENT_DIR + "tsfiles/"; + // private static final String RESULT_CSV_PATH = OUTPUT_PARENT_DIR + "write_time.csv"; + private static final String RESULT_CSV_PATH = OUTPUT_PARENT_DIR + "write_time2.csv"; + private static final String READ_RESULT_CSV_PATH = OUTPUT_PARENT_DIR + "read_time.csv"; + // private static final int REPEAT_TIMES = 100; + private static final int REPEAT_TIMES = 200; + private static final String DEVICE_NAME = "device_1"; + private static final String MEASUREMENT_NAME = "sensor_1"; + private static final int MAX_DECIMAL_PRECISION = 8; + + private static final List ENCODINGS = + Arrays.asList( + TSEncoding.TS_2DIFF, + TSEncoding.RLE, + TSEncoding.GORILLA, + TSEncoding.CHIMP, + TSEncoding.SUBCOLUMN + ); + + private final IDeviceID deviceID = Factory.DEFAULT_FACTORY.create(DEVICE_NAME); + + @Test + public void benchmarkCsvWriteEncodings() throws Exception { + File inputDir = new File(INPUT_PARENT_DIR); + File outputDir = new File(OUTPUT_PARENT_DIR); + File tsFileDir = new File(TSFILE_OUTPUT_DIR); + if (!inputDir.exists()) { + throw new IOException("Input dataset directory does not exist: " + INPUT_PARENT_DIR); + } + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + if (!tsFileDir.exists()) { + tsFileDir.mkdirs(); + } + + File[] csvFiles = inputDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".csv")); + if (csvFiles == null || csvFiles.length == 0) { + throw new IOException("No CSV dataset files found under " + INPUT_PARENT_DIR); + } + Arrays.sort(csvFiles, Comparator.comparing(File::getName)); + + CsvWriter writer = new CsvWriter(RESULT_CSV_PATH, ',', StandardCharsets.UTF_8); + writer.setRecordDelimiter('\n'); + writer.writeRecord( + new String[] { + "Dataset", + "Encoding Algorithm", + "Write Total Time Nanos", + "Dataset Read Time Nanos", + "Write CPU Time Nanos", + "Write IO Time Nanos", + "Write IO Write Nanos", + "Write IO Flush Nanos", + "Write IO Force Nanos", + "Points", + "Max Decimal Precision", + "Multiplier", + "TsFile Size Bytes", + // "TsFile Path" + }); + + try { + for (File datasetFile : csvFiles) { + DatasetProfile datasetProfile = analyzeDataset(datasetFile); + int boundedPrecision = Math.min(datasetProfile.getMaxDecimalPrecision(), MAX_DECIMAL_PRECISION); + long multiplier = getMultiplier(boundedPrecision); + + String datasetName = extractFileName(datasetFile.getName()); + System.out.printf( + "Writing dataset=%s, points=%d, precision=%d%n", + datasetName, datasetProfile.getPointCount(), boundedPrecision); + for (TSEncoding encoding : ENCODINGS) { + System.out.printf(" Encoding=%s%n", encoding.name()); + java.nio.file.Path tsFilePath = + Paths.get(TSFILE_OUTPUT_DIR, datasetName + "_" + encoding.name().toLowerCase() + ".tsfile"); + WriteBenchmarkResult benchmarkResult = + benchmarkWrite(tsFilePath.toFile(), encoding, datasetFile, multiplier); + + writer.writeRecord( + new String[] { + datasetName, + encoding.name(), + String.valueOf(benchmarkResult.getTotalTimeNanos()), + String.valueOf(benchmarkResult.getDatasetReadTimeNanos()), + String.valueOf(benchmarkResult.getCpuTimeNanos()), + String.valueOf(benchmarkResult.getIoTimeNanos()), + String.valueOf(benchmarkResult.getIoWriteNanos()), + String.valueOf(benchmarkResult.getIoFlushNanos()), + String.valueOf(benchmarkResult.getIoForceNanos()), + String.valueOf(datasetProfile.getPointCount()), + String.valueOf(boundedPrecision), + String.valueOf(multiplier), + String.valueOf(benchmarkResult.getTsFileSizeBytes()), + // tsFilePath.toString() + }); + } + } + } finally { + writer.close(); + } + } + + @Test + public void benchmarkTsFileReadEncodings() throws Exception { + File tsFileDir = new File(TSFILE_OUTPUT_DIR); + if (!tsFileDir.exists()) { + throw new IOException("TsFile directory does not exist: " + TSFILE_OUTPUT_DIR); + } + + File[] tsFiles = tsFileDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".tsfile")); + if (tsFiles == null || tsFiles.length == 0) { + throw new IOException("No tsfile files found under " + TSFILE_OUTPUT_DIR); + } + Arrays.sort(tsFiles, Comparator.comparing(File::getName)); + + CsvWriter writer = new CsvWriter(READ_RESULT_CSV_PATH, ',', StandardCharsets.UTF_8); + writer.setRecordDelimiter('\n'); + writer.writeRecord( + new String[] { + "Dataset", + "Encoding Algorithm", + "Read Total Time Nanos", + "Read CPU Time Nanos", + "Read IO Time Nanos", + "Read IO Read Nanos", + "Points", + "TsFile Size Bytes", + // "TsFile Path" + }); + + try { + for (File tsFile : tsFiles) { + ReadBenchmarkResult benchmarkResult = benchmarkRead(tsFile); + String fileName = extractFileName(tsFile.getName()); + int splitIndex = fileName.lastIndexOf('_'); + String datasetName = splitIndex >= 0 ? fileName.substring(0, splitIndex) : fileName; + String encodingName = splitIndex >= 0 ? fileName.substring(splitIndex + 1) : "unknown"; + + System.out.printf("Reading tsfile=%s%n", tsFile.getName()); + + if (datasetName.endsWith("_ts")) { + datasetName = datasetName.substring(0, datasetName.length() - 3); + } + + if (encodingName.equals("2diff")) { + encodingName = "ts_2diff"; + } + + writer.writeRecord( + new String[] { + datasetName, + encodingName.toUpperCase(), + String.valueOf(benchmarkResult.getTotalTimeNanos()), + String.valueOf(benchmarkResult.getCpuTimeNanos()), + String.valueOf(benchmarkResult.getIoTimeNanos()), + String.valueOf(benchmarkResult.getIoReadNanos()), + String.valueOf(benchmarkResult.getPointCount()), + String.valueOf(benchmarkResult.getTsFileSizeBytes()), + // tsFile.toPath().toString() + }); + } + } finally { + writer.close(); + } + } + + private WriteBenchmarkResult benchmarkWrite( + File tsFile, TSEncoding encoding, File datasetFile, long multiplier) + throws IOException, WriteProcessException { + long totalTimeNanos = 0; + long totalDatasetReadTimeNanos = 0; + long totalCpuTimeNanos = 0; + long totalIoWriteNanos = 0; + long totalIoFlushNanos = 0; + long totalIoForceNanos = 0; + + for (int repeat = 0; repeat < REPEAT_TIMES; repeat++) { + // System.out.printf( + // " Write repeat %d/%d, file=%s, encoding=%s%n", + // repeat + 1, REPEAT_TIMES, datasetFile.getName(), encoding.name()); + Files.deleteIfExists(tsFile.toPath()); + WriteBenchmarkResult singleRunResult = writeTsFile(tsFile, encoding, datasetFile, multiplier); + totalTimeNanos += singleRunResult.getTotalTimeNanos(); + totalDatasetReadTimeNanos += singleRunResult.getDatasetReadTimeNanos(); + totalCpuTimeNanos += singleRunResult.getCpuTimeNanos(); + totalIoWriteNanos += singleRunResult.getIoWriteNanos(); + totalIoFlushNanos += singleRunResult.getIoFlushNanos(); + totalIoForceNanos += singleRunResult.getIoForceNanos(); + } + + long tsFileSizeBytes = Files.size(tsFile.toPath()); + return new WriteBenchmarkResult( + totalTimeNanos / REPEAT_TIMES, + totalDatasetReadTimeNanos / REPEAT_TIMES, + totalCpuTimeNanos / REPEAT_TIMES, + totalIoWriteNanos / REPEAT_TIMES, + totalIoFlushNanos / REPEAT_TIMES, + totalIoForceNanos / REPEAT_TIMES, + tsFileSizeBytes); + } + + private ReadBenchmarkResult benchmarkRead(File tsFile) throws IOException { + long totalTimeNanos = 0; + long totalCpuTimeNanos = 0; + long totalIoReadNanos = 0; + long pointCount = -1; + + for (int repeat = 0; repeat < REPEAT_TIMES; repeat++) { + // System.out.printf(" Read repeat %d/%d, file=%s%n", repeat + 1, REPEAT_TIMES, tsFile.getName()); + ReadBenchmarkResult singleRunResult = readTsFile(tsFile); + totalTimeNanos += singleRunResult.getTotalTimeNanos(); + totalCpuTimeNanos += singleRunResult.getCpuTimeNanos(); + totalIoReadNanos += singleRunResult.getIoReadNanos(); + pointCount = singleRunResult.getPointCount(); + } + + return new ReadBenchmarkResult( + totalTimeNanos / REPEAT_TIMES, + totalCpuTimeNanos / REPEAT_TIMES, + totalIoReadNanos / REPEAT_TIMES, + pointCount, + Files.size(tsFile.toPath())); + } + + private WriteBenchmarkResult writeTsFile( + File tsFile, TSEncoding encoding, File datasetFile, long multiplier) + throws IOException, WriteProcessException { + ProfilingTsFileOutput profilingOutput = new ProfilingTsFileOutput(tsFile); + long encodeAndWriteTimeNanos = 0; + long datasetReadTimeNanos = 0; + + try (TsFileWriter tsFileWriter = new TsFileWriter(profilingOutput, new Schema())) { + tsFileWriter.registerTimeseries( + new Path(deviceID), + new MeasurementSchema(MEASUREMENT_NAME, TSDataType.INT32, encoding)); + try (InputStream inputStream = Files.newInputStream(datasetFile.toPath())) { + CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); + long timestamp = 1L; + try { + while (true) { + long readStartNanos = System.nanoTime(); + boolean hasRecord = loader.readRecord(); + datasetReadTimeNanos += System.nanoTime() - readStartNanos; + if (!hasRecord) { + break; + } + + String value = getFirstColumnValue(loader); + if (value == null) { + continue; + } + + long writeStartNanos = System.nanoTime(); + TSRecord record = new TSRecord(deviceID, timestamp++); + record.addTuple(new IntDataPoint(MEASUREMENT_NAME, scaleValue(value, multiplier))); + tsFileWriter.writeRecord(record); + encodeAndWriteTimeNanos += System.nanoTime() - writeStartNanos; + } + } finally { + loader.close(); + } + } + } + + long cpuTimeNanos = Math.max(0L, encodeAndWriteTimeNanos - profilingOutput.getIoTimeNanos()); + long totalTimeNanos = datasetReadTimeNanos + cpuTimeNanos + profilingOutput.getIoTimeNanos(); + return new WriteBenchmarkResult( + totalTimeNanos, + datasetReadTimeNanos, + cpuTimeNanos, + profilingOutput.getIoWriteNanos(), + profilingOutput.getIoFlushNanos(), + profilingOutput.getIoForceNanos(), + Files.size(tsFile.toPath())); + } + + private ReadBenchmarkResult readTsFile(File tsFile) throws IOException { + ProfilingTsFileInput profilingInput = new ProfilingTsFileInput(tsFile); + long startNanos = System.nanoTime(); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(profilingInput); + TsFileReader tsFileReader = new TsFileReader(reader)) { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(deviceID, MEASUREMENT_NAME, true)); + QueryExpression queryExpression = QueryExpression.create(paths, null); + QueryDataSet queryDataSet = tsFileReader.query(queryExpression); + + long pointCount = 0; + while (queryDataSet.hasNext()) { + RowRecord rowRecord = queryDataSet.next(); + if (!rowRecord.getFields().isEmpty() && rowRecord.getFields().get(0) != null) { + pointCount++; + } + } + long totalTimeNanos = System.nanoTime() - startNanos; + long cpuTimeNanos = Math.max(0L, totalTimeNanos - profilingInput.getIoTimeNanos()); + return new ReadBenchmarkResult( + totalTimeNanos, + cpuTimeNanos, + profilingInput.getIoReadNanos(), + pointCount, + Files.size(tsFile.toPath())); + } + } + + private static int getDecimalPrecision(String str) { + int decimalIndex = str.indexOf('.'); + if (decimalIndex == -1) { + return 0; + } + return str.substring(decimalIndex + 1).length(); + } + + private static DatasetProfile analyzeDataset(File datasetFile) throws IOException { + long pointCount = 0; + int maxDecimalPrecision = 0; + + try (InputStream inputStream = Files.newInputStream(datasetFile.toPath())) { + CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8); + try { + while (loader.readRecord()) { + String value = getFirstColumnValue(loader); + if (value == null) { + continue; + } + + pointCount++; + maxDecimalPrecision = Math.max(maxDecimalPrecision, getDecimalPrecision(value)); + } + } finally { + loader.close(); + } + } + + return new DatasetProfile(pointCount, maxDecimalPrecision); + } + + private static String getFirstColumnValue(CsvReader loader) throws IOException { + String[] values = loader.getValues(); + if (values.length == 0) { + return null; + } + + String value = values[0].trim(); + return value.isEmpty() ? null : value; + } + + private static long getMultiplier(int decimalPrecision) { + long multiplier = 1L; + for (int i = 0; i < decimalPrecision; i++) { + multiplier *= 10L; + } + return multiplier; + } + + private static int scaleValue(String rawValue, long multiplier) { + return new BigDecimal(rawValue).multiply(BigDecimal.valueOf(multiplier)).intValue(); + } + + private static String extractFileName(String path) { + File file = new File(path); + String fileName = file.getName(); + int dotIndex = fileName.lastIndexOf('.'); + if (dotIndex <= 0) { + return fileName; + } + return fileName.substring(0, dotIndex); + } + + private static final class WriteBenchmarkResult { + private final long totalTimeNanos; + private final long datasetReadTimeNanos; + private final long cpuTimeNanos; + private final long ioWriteNanos; + private final long ioFlushNanos; + private final long ioForceNanos; + private final long tsFileSizeBytes; + + private WriteBenchmarkResult( + long totalTimeNanos, + long datasetReadTimeNanos, + long cpuTimeNanos, + long ioWriteNanos, + long ioFlushNanos, + long ioForceNanos, + long tsFileSizeBytes) { + this.totalTimeNanos = totalTimeNanos; + this.datasetReadTimeNanos = datasetReadTimeNanos; + this.cpuTimeNanos = cpuTimeNanos; + this.ioWriteNanos = ioWriteNanos; + this.ioFlushNanos = ioFlushNanos; + this.ioForceNanos = ioForceNanos; + this.tsFileSizeBytes = tsFileSizeBytes; + } + + private long getTotalTimeNanos() { + return totalTimeNanos; + } + + private long getDatasetReadTimeNanos() { + return datasetReadTimeNanos; + } + + private long getCpuTimeNanos() { + return cpuTimeNanos; + } + + private long getIoTimeNanos() { + return ioWriteNanos + ioFlushNanos + ioForceNanos; + } + + private long getIoWriteNanos() { + return ioWriteNanos; + } + + private long getIoFlushNanos() { + return ioFlushNanos; + } + + private long getIoForceNanos() { + return ioForceNanos; + } + + private long getTsFileSizeBytes() { + return tsFileSizeBytes; + } + } + + private static final class DatasetProfile { + private final long pointCount; + private final int maxDecimalPrecision; + + private DatasetProfile(long pointCount, int maxDecimalPrecision) { + this.pointCount = pointCount; + this.maxDecimalPrecision = maxDecimalPrecision; + } + + private long getPointCount() { + return pointCount; + } + + private int getMaxDecimalPrecision() { + return maxDecimalPrecision; + } + } + + private static final class ReadBenchmarkResult { + private final long totalTimeNanos; + private final long cpuTimeNanos; + private final long ioReadNanos; + private final long pointCount; + private final long tsFileSizeBytes; + + private ReadBenchmarkResult( + long totalTimeNanos, + long cpuTimeNanos, + long ioReadNanos, + long pointCount, + long tsFileSizeBytes) { + this.totalTimeNanos = totalTimeNanos; + this.cpuTimeNanos = cpuTimeNanos; + this.ioReadNanos = ioReadNanos; + this.pointCount = pointCount; + this.tsFileSizeBytes = tsFileSizeBytes; + } + + private long getTotalTimeNanos() { + return totalTimeNanos; + } + + private long getCpuTimeNanos() { + return cpuTimeNanos; + } + + private long getIoTimeNanos() { + return ioReadNanos; + } + + private long getIoReadNanos() { + return ioReadNanos; + } + + private long getPointCount() { + return pointCount; + } + + private long getTsFileSizeBytes() { + return tsFileSizeBytes; + } + } + + private static final class ProfilingTsFileOutput extends OutputStream implements TsFileOutput { + private final FileOutputStream outputStream; + private final OutputStream bufferedStream; + + private long position; + private long ioWriteNanos; + private long ioFlushNanos; + private long ioForceNanos; + + private ProfilingTsFileOutput(File file) throws IOException { + this.outputStream = new FileOutputStream(file); + this.bufferedStream = new NoSyncBufferedOutputStream(new TimedOutputStream()); + } + + @Override + public void write(int b) throws IOException { + bufferedStream.write(b); + position++; + } + + @Override + public void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(byte b) throws IOException { + bufferedStream.write(b); + position++; + } + + @Override + public void write(byte[] b, int start, int offset) throws IOException { + bufferedStream.write(b, start, offset); + position += offset; + } + + @Override + public void write(ByteBuffer byteBuffer) throws IOException { + int remaining = byteBuffer.remaining(); + if (byteBuffer.hasArray()) { + bufferedStream.write( + byteBuffer.array(), + byteBuffer.arrayOffset() + byteBuffer.position(), + remaining); + byteBuffer.position(byteBuffer.limit()); + } else { + byte[] bytes = new byte[remaining]; + byteBuffer.get(bytes); + bufferedStream.write(bytes); + } + position += remaining; + } + + @Override + public long getPosition() { + return position; + } + + @Override + public void close() throws IOException { + bufferedStream.close(); + outputStream.close(); + } + + @Override + public OutputStream wrapAsStream() { + return this; + } + + @Override + public void flush() throws IOException { + bufferedStream.flush(); + long ioStartNanos = System.nanoTime(); + outputStream.flush(); + ioFlushNanos += System.nanoTime() - ioStartNanos; + } + + @Override + public void truncate(long size) throws IOException { + bufferedStream.flush(); + outputStream.getChannel().truncate(size); + position = outputStream.getChannel().position(); + } + + @Override + public void force() throws IOException { + flush(); + long ioStartNanos = System.nanoTime(); + outputStream.getFD().sync(); + ioForceNanos += System.nanoTime() - ioStartNanos; + } + + private final class TimedOutputStream extends OutputStream { + + @Override + public void write(int b) throws IOException { + long ioStartNanos = System.nanoTime(); + try { + outputStream.write(b); + } finally { + ioWriteNanos += System.nanoTime() - ioStartNanos; + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + long ioStartNanos = System.nanoTime(); + try { + outputStream.write(b, off, len); + } finally { + ioWriteNanos += System.nanoTime() - ioStartNanos; + } + } + + @Override + public void flush() throws IOException { + outputStream.flush(); + } + + @Override + public void close() throws IOException { + outputStream.close(); + } + } + + private long getIoWriteNanos() { + return ioWriteNanos; + } + + private long getIoTimeNanos() { + return ioWriteNanos + ioFlushNanos + ioForceNanos; + } + + private long getIoFlushNanos() { + return ioFlushNanos; + } + + private long getIoForceNanos() { + return ioForceNanos; + } + } + + private static final class ProfilingTsFileInput implements TsFileInput { + private final TsFileInput input; + + private long ioReadNanos; + + private ProfilingTsFileInput(File file) throws IOException { + this.input = new LocalTsFileInput(file.toPath()); + } + + @Override + public long size() throws IOException { + return input.size(); + } + + @Override + public long position() throws IOException { + return input.position(); + } + + @Override + public TsFileInput position(long newPosition) throws IOException { + input.position(newPosition); + return this; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + long ioStartNanos = System.nanoTime(); + try { + return input.read(dst); + } finally { + ioReadNanos += System.nanoTime() - ioStartNanos; + } + } + + @Override + public int read(ByteBuffer dst, long position) throws IOException { + long ioStartNanos = System.nanoTime(); + try { + return input.read(dst, position); + } finally { + ioReadNanos += System.nanoTime() - ioStartNanos; + } + } + + @Override + public InputStream wrapAsInputStream() throws IOException { + return input.wrapAsInputStream(); + } + + @Override + public void close() throws IOException { + input.close(); + } + + @Override + public String getFilePath() { + return input.getFilePath(); + } + + private long getIoReadNanos() { + return ioReadNanos; + } + + private long getIoTimeNanos() { + return ioReadNanos; + } + } + +} \ No newline at end of file diff --git a/optimal_pack_prune b/optimal_pack_prune new file mode 100755 index 0000000000000000000000000000000000000000..ea39497d14d236dd3e1b7a9596975415c6015a40 GIT binary patch literal 61040 zcmeHw3tUvy+V|Rf2Aly@ltC_D7}N~VxQQs8iUCCN5-L!cbsDb92m-?3T^TPGigg^d z8+H;)%ON#2Eje`(yS*CL@zUv3r|*bnIf#|ujWKDy|FdTe4kMP?`}@7$?|Wz0^1q(7 z*Y#P?de&OcTFm~()gONGVoYFoM7Z8?=jDt&&Vnh31;C}iX|}SL_6lE;Q4`v(xcVp6c?3Uub^{&H3_!-hI!lK z6AgCX^wDZ(-${1|vkyLs}-eS3*;QBb2 zI4*TU^5}7PzSKL_3p70?;WniD@TN-vF{ruJ_4TMagl0IWHHR7VM?|WQKV0RfjLiqC z|HYVD#h54jli}zI<&MS$6*B?uW{>|r8C%-L%N__*!;#!Zl$Vu86zZ}ga_1N3f~Kc0 z9Mz-y(|K<{nHK%^&avr}p8a&UHX31SI8+r+ASxUv`HA;T_)H!K;G#_-EClVNd}tgx zK2&Zb9KDmeAn8aq%C(~|s87z&q2W>CsFW^Ss^=)j_-vvy(jDWc&o9&G%?eNCNf4W{ zA#hZmso*D-N4z7N_)AZJIHJAJF_vkU1$yxtPxH`yvzi78v#u43rmHh#rZ%nIbon7S z)5S^druK#1Ozj^CM(YPYM=!{P+KVz)3mv*f(W0>^1dU~@%)gC!7}7rN!PYl!U>5Lb z4b#=%3*4zV(A(H_h8fi@f4PUbC2VJMVY@O|V^OM^MG!RG!T0S!Y9Wq!8q)qw>Hosh z&#J%I{3J6nq?7Xb&c|p^&wAD0%e)QuDqBXkDbLH>n41RqBA-WvxQ!lc_E3RkWVvfB zjUUL38OW2u3=Cl&2+NThCn3zd|HHVZzi2E$0-J3{n0i-b^`?6bX@}@t&_{7i8~8iq zLGM0B_!YD}Q_l8S$Fe%}au1_oDKnb=n8m!@&Dh6{9Zeygm^7Agz|c(^%Xr}UEgFlp z!rNr6Vir&EO3__oF~8w%8VY}ir0)HQ4mdx#n5f>sEIwPx#2twBuD zymbr9xP+uIZE-Rhy)9rS0`5-KyK-R*|9$O3C4k%t9o zQe+9cFAeHxYMcwXLf(zCfZ9_s-&*SH_<|9m%+?R$eEPHA3ZZReg*%&Fhq|gomQexO zNB}mW90kf~TBxzayRjB}ukmCV@k-W`0C{RcJ3y)PfvLN-z@65Am$2loa;&E;F&iuewyyZNHhCFp_!j3Ji z1v?J6Hq5g1^Cz>OK6dv0&UtFwS%xxQ6BjD6Ad|l%|au zi&n;FkAN7~CymvB+GOYy8vh{}tL8@pOJllV z`S~HiLSqf{r=%|~|6=tcy|L~;@v$xNTM^HSI99}MgYNnHgh!mO!1kD-pQ#?C+XU#g zMw!1f-qTS={s7d|&5))n7%@-M?`ZFV-IdkLc0X55`s=V4ThCbiy$S{NX&JLLjx^jG zbd8yU7_*p9d=hu%kt9<-Ym046Pq6qRKlA?4q&s|?)@dviqOI+F9NNB7gzhr1jE#@6 zmJcv4DjoEcz3uK@wEZG)I~EEm=XPGSw-a*|mDz+j%CVgRR4=qqfi{}0Nj81|MygHU zqrDya9_?M@*xnqpH==WU?QQ)U@uWGWgwH8EeT~`B{psCM)~}Q?0q>{ypTTGv{CeFZiN1=Y&|h*eKtE< zX|={0M2)2u?=4oV--Rd9M&xlaNoEvL|1EOVAAPC7eAncMc~_Ue&qoNDxwJ3KxcJay z(|%DU<+m02?dAE^AirmMel*`-M4r|Oki)TXlLQONt2qMl^Iyzj$Y#d*RL5`KjP^Bv z5X_8|5oS0g7?%L4d}+Q#oU^E#Mo?A-^E7sT1cG+v2l0{e&7!!}ra>M?n%5L=JTBU# zU>LJYHE}^`Qx^0##iw}G_HAgpwW5#7`VPZ1$y6If&*r3VJkD}A<1)z5CB%`+b;_$A zJgz_*ly%g6!Y!@|@#A$Nqs%8BkHb9rwz=IJb7*G%sQ3|VcH=MBjQFK&b`22gPu1DC zHI}p2tb3Z?)L4cJ9#sV`+EEpVmq~dd9?gX*l<(WjV$ZX(-TGE5`j5(Krt~sJEtasV zW|>E=iW!a#)9FWTMA}WrV?D;7R!}u(S*;lt@jk_39ZBObk>+8fr?y^3*=Ct%t*@YJ zu|Wyt4Jg05$8kGXR0q{4<NZOmI*tsD)201JpU(a zY$M9}68X$SKbTK^6nF7kH>0EODkuFw3&z7#(=rvzv3#-ApIGQ z;4uPpgu}Nd0v9lCmhL_!q zZvcO@Oi9tA=`JkV_a?Ix-a5>o9@Gyx9h$;@V+ZOl*G$}&N@%)-FrYg zk2|{_T267f9@27uCNNQi&bIoQtQh~7Crvh;fKF0lT$#aBCgcp*h&cITp9V#H(z>P# z!fxFI&$KEQD>o?H7aw0aiS1QrG6(p!EIv+oS0FBxcM|^R;3rx4h4W$O*VaSM1o)>S zoGCL0C|K@GiiIQEDj>&YnA7%R99$kW*`yTuG=!q8vnXpO$_ifWu>rc#xL*{Sziqc> z1POhb4Uh-Y4bMS-D2)PP^;m;A*3koXH1EgUiZ$@K$2V4Md)e0Fo~Q%%_Nud3E8GLj z?4d~2Ca{)ld+_J*y@>uGv>0_)PFjX8S_X?xl!FW4jM{etGC*lRw%Q2p0f{wx*?k*N!~% z1v$SH&SV`|Q6*n`$u6G*_@XE%>cQ-qgQ+;C$fx42Cq= zC{_8Qe?!|t8jF8VMHOVok|I+y1PcAC%<0&J3BC;hLO|6Q=<6$@w{gAbebhWbW4QqR zcqxf=jc;@IHEYJzGfGq2L(H;Y^lRQ1{akfE;*!2IPhgh6MORi^(60+GFF8(Yjg%`u zj73^o`3ilT4d6+6T-)0ukN@_7%;Q}n-t}X?Ro4(dOBPT?^UlwBcMaihQ=GB3dA0|} z8T8HUfDbnYJ|gpJ$d?6pV$PW2IA1Ede-(JJ2aEAptXX$qMBB)6)Pu^Zr~DxM{xVh7 zw4vau!l$YcbF3M2ERBcbc&`ATps$XtmibmepIIhA#?E0b9#*mB_z%$i;pmf}u-~J3 zM+Scd>QaKbc=xWK)=O?kixj;ltXUXJeQPWbysCZzFF%7*2muY-LB|VNPbgW2Rq$^< zAN727kl3SHSoU@+Jr4S5!TBAg6vg{orv z6aC)bX2`8c-ITj2_5IwJQ@_o9B{c_m1w)n?!WhCB!r0cGsWLWXKoAp~8?Ru@%a^p$ z9yb_e48FKztpMjO3RRh2^3+$9VpSrOH)P2V+@nU`iQa72Cd#iP51}~=V|zQ4oVO@e zY37&gLt6`E^~(djsoi3)<{ZS|LVA9=oALVEB%ro8)_-l29yEpXz8waX)-i(~G-1!# zV8)zv3hR?^(PyV+eQV!`O=1w@&_15#pR1EJ7OWHYG``L(BM{z*xksY0R)``R^NB=5 z2JRvnGQqyL?jd;%1$`IlB-Q5w(I~QE{yULq!uoHN34MUQ%}fLKLu$vlC9;q4 zWy~$l(wZ^IKDSKZb`-H=Zh0E-NoUi%A+77s9ll6W1_bqO^c-Fa>pfl=5FMsubu^dL`g0}frBpHtOUN9s8EuiuFR|4(sf+SM zr1Aqg%4aAqG$3<;8tKhkw=p4azRJ$DL=Zy9#k78yT(V@s7b3)v=dAU#V%otk%>C;Kx$Dmt@Jd ze_9i)_m^yPGUP#gdCZ>o=l;flkjr)%;v;;jJ-lCl@qNf1?2PNFE@=qwO2&K;?kHnk zpl>E)=@ggCSOoV+0-IjZbgacLpdC7_#WDvnV}FVRT3U;3v8~1KLRjW9cjH8QhdD$V zdmk`MDrj@i2XO}(*0UQwpgqCJ3)m;VnHe7|{){pBh{lpkaqun$d$h&^*b+fsq45d( zgBJb^%QcqaBHMHENm|?b)Y9H=DfEs9#=_0^c~pPe=QScN?e&I%##}1d9;Njr_RM!8 zjO@4)4O_t=&|76w+h8w09}2rQ*}dWgMe|bVM=BS#ZDaewv9NcKf!#Y9_V3ZKgD07; z6(^dS1|`gD9(u5_UZ(vb34PNYbxQ!!di|=%Ag)Z+2BdxbV3czSezLP&0Mc4Jmin2g zKff1#9Ji_NgQNBQ{cvQ9y@WR4+(7bEzkUyWIF!={j@YY6@5w%q-_vHBnCB3d!o%kB zuu_hCdz^}o;NfTtmx?496;tz7i^L4;GfUT{+!!bv|l>)wI9P3;RuqwYy)Ln7LefHC~hRvNc-c0p@%LK>%plVUSc>|v`5wU?odpCZjt zp5F>h``=pIhSfIf>Ut;$S0?c>TG=>myfQzLD3M(@-BMZ2bNT zWp~N%FTn3aW_$_nZ{Rn=mx=wOja2WO@cZMAHvJBMBj3l7&d%>j_%pG`e@n_sRht34 z8DB1Q8Ki!@8+|Yhd}<43Y6@y4%qEy}ZU zSu{fyQy`0#ki{47W0tSbe#rHaD{q0)j3ael?KQk<8N?DD*MT6VWeGYRtcieyn$p5K};VP@y05p?%>w>?x@Z*z+C@X}891 z#d&TJ_GbsM&#gp#X$(V$q|rQjLS!u|@E?K;5h??9s7t7%ACy%}tj&T2MOYx`ku%@H zCnMODWJ2y>9QU#cU=#L@v*@n-e!VC(;>_ZnLB{Bu>$>} ze{i&k(yaU?Xe6x}^T7v&MYC3b-5C4lv`o~8%1B{tZ{-i6wkWFf$3zp(e=PaS5=fRy z8zEm?f6*RK!5Y)NJZrZJayyRIw{AsT<>Euw11TFq+)}C(?T;N-F3p0uVP|igfi;*! zMZ+r5qe=zawfV`c-6OEh*9S67Ip*3*)HNUe_JxUPdjjT~4JaDpAHCSUB0o%0cm+N-Bk+9n}VSi0)(}S=lUgq;| zGk93oJ)vzi;;)9CZx!NAv{W45h;g=Z>B{PJ7ufD?wT(Dy^lBazRaxEahczW^&p|ks z+ln(lMK}K{276;J>Sab=v@XT?Ii|!}dgDOY{FXD5{~+at#%~4FL5w@K#hJ((>pk1r1Ue4uWJ#V6lil(Lc@t~3 zQ&~$sbdArWERJkI=ARbtCi`e2`u?&cdqQo#!uYtZ%fzGY-9Xbdz$UvwJVB;yQn+Y@<`9-#JPeMj5>j!LO9b*BqcA%4tG3*mBBHpubQW-C5Y-RM*SlSWh zJl<8CHLzjeyaYOy`h)u6KZxh7Cz>!$gAlI~=Qtr*>?rA((3TUU#$o>k9d0>_G~lJ> zGRC95z7kJVeU0y0tx(EA_ z3@bllyi&p9ENBne3(O9B$bN^(iu&5`FkKlWm>Rn=*g^s%c{~NV3`V=+3r3DISJcNf zb+he@%*ZbocQ88F$xo^i#jAw=4q|@I2WWjosR%J!*!7-(HC9&H*V z!<|Dp_GxYy6YiJ`x?xUGV6IeR&m|f?)Ebk&;L+e`O{hhyr@pr0y%p~jLjC}{t7vv-Q_MYCT`lyH=kmnkDc~R02bLdt zYAii)ZfQh%hPtWcY?2@sCPjj#ej`~75_&ZE*!6U^@9t-$vxKPLbe@1Yr$5UG61`it zLJkq{h$6U~QN+1JqQGKO7HQTg+mnumez$n7NvCTIx#M7UeqT0{&Qj>CpXxgZ&fHT| zrx3h7<=u}2J|SmQHsUO(yBEvoi8eszW4~ZHrntOhZ5z(l0)AYyHt_7CwSC=}tnJsj zWbH4InR7A?)&btq9AcI!!rb~OHee6Z>nFjxDwug!trmJ!{YTcT>V)WCkcl&+px!ol zqP@r&Sx?%Vs)7=)hbd9sV5%Qv%ZxF7M&?yZ=Q?sYH@NTnRct>$`u%Mm;|%5uocHva z)F`-*Q)-INzK-ZqZxxu*kGxkBx&kltghC&W;HZTEe)`E}&m4RpESiiDv z%_p&jcmwf?=LZ*g_dO#jCQyHvwhn zlnoI$o3@{+*?I5Yzhb+0@*n5@Prt(Zc-E&GfAEnJkF&T;ta&1E21mR;Cyfb*SLp5Q?+yk40reSAFy`&g>GP5A1yu&w5IIM0U}` zq8Iw8@TPsV8vNOaa+=XclAQa#w!!R)wT18C4fzWq+rB;3ZKUq&>^9oRKZSdNieHkB zD=_v#!S~SMBh}k<`E4r&WvxjU($*7w+=y{cYszuZ8D&748^*v-d-?W-KFxy;Ua1{Gi1A;DI!rNb7|>x+AY1m>+uL zPP)u^s+(vcy%UCc!V~)`I@b?{u5aqeEIqMiY$E@i&};i;G_Phf_p;3s%N*v32=MFw zr}<$UV=m^A&<$g9rAqTFBZsX|3bk7HIL#)=AXGKljrF2al?K!AEj%AsV2kz%gp|tzKyF<{=Tfw`7 zLw=oag${hnK^jMXedu!|zdX=?w*Aq0X@6wro9-6*w%mblephjp0e(J&Fo{oSf76}P zo%q?Wcema3X~KR#3_7c65ZT!HFN zrU^RY>`?3{1&;}5kq6y93*EIUunB+bKy%6K9xS6T`j_r#()nC-UuGFPzhIvS&I-l_ zHw0EJhWs5^Ijv~|t80FQ)#=bzO-ZbdWOQLSlsOz_%9PEgaQ~v|5!`Jq^16Ad#-^Z0j%ES>3;o{}tp<>rIlQk5CTH z1%IZt-1K~j;cQ7!tLUex3+j)%q0kjCz}`Cqc~=K&>RxymwhOL9?#39yJ#E!9Sa-(C zd>anAvt5IslZc0OPs$T|q_fT7{MwPQ8NAK)!DiHh_CeGZy4MvT>r>??^Q)qK?dRmQ z=XnqDo}bQ4&tcC`Hb0uj)*|c$*bgd}Fcbc6#6oq5k@+@fVhy7b>F$)Ob~WN)?G5|5 zs%j46*!EDW+FFFqLE4+y2e$ugeePQE-_QpF`a(pX$j~=9e_IHcXG@>ZnX^S zS_9JK%zO4>)Y0CKmw7v={-=>o3+hho!JpYmZTbOp`@&@O^=R~Y68b(7GLV3A__1W~ zBb(w?j9nT}0@)G`eT_IDBD>DcV6|jd+?fbF5REY_!uvvw>}#n@H0B|bOMrG;-IMT# z(A@sfjkZOF9A_VbP^H9K;z7uO2kh)a;bsaRpJ)Z4 zJ3q9|*)F?O!S*N-*8}l}clW4L`m)|4?(rUie0nGhY2yUiF9^+HQA?}QmuvHb4`RLR z(V)BIP&M|mtLO{|SeyR@YYS!`4KdeH9<$Y9nY6FPnKf)>CxZ-WU!hGj&O%h0)_VH; z7VPc78NaXWkX2okUFldh#@*`vqrS*1n%u>pjXHybr64-@CE)Zfov%C;7bY0r2EFS_OAiG4*I45Va{dX&f!k) z?{U^927Mv<5x2t+{o3J&J-5S;^Xv|Ppo4#)gMWyFKg_`&?%wxI!gdg;L&Lyv*}fqw3Ip!1{$^MLeV ztsy>I>A~NbN{=`G zrMS2YB|W&$BRxL&m(r+gB=hCi2S{oCIPDLly1>3DA@WD;E1)7^xVl2I$rvl_fSrU0jVuR;iyd5O{C{8j^RKm z8+&1?UeFiP0~@MThiFa@=Lnms^o-yb%W))->V&%((i4Y&sot=QF{<02;OM!>2JBuS z)e-l=Bwoeya1D^^n!xEqj!8hO^Jq>da~uPtx{u|wUCzdFdOXJo947*)E%$R8HYVwr zgny|`Tt3+pTQE!Gn9ebS;{!lyHpn9DJb<8+QQfW(u0PU|?%pXT@%j%zp?Ij-fnj^lbD^~E!sewO2N9G~a-0>^5O8#unm@vj_T;<%Ax4aZF! zU*`A<$ITqKaC{X=efJutw{kQAqu^>e{W`~O9Jh1a!SM|s_32Jd@8Y|Y$e$6~j_w@0 zaa3?r0!fBFIE`~$>G9&&onsGpiB7|U@a$59;PINr_iPaN;zcrVBMIKpl$J=pt5PXb4r0ZLC2$I%>< zIga5tmSYOXaU92UoWOA+N1Q)NPb$Yr94B*}!ZD3wI!71?q~`&S4|05n<5Z3hbDYNU z5sr^?gk4N}9^(i*i1cJ~%;K2MF^6L=$2^YHIl|T>J^37U9A|PY;8+Nxu{?{@MI4Jc zmT;WSv6N#OM?J@Kj&nH9p4Ed@mY?~aeSWR3mmICZs7PL z$G>uXiQ`6&H5@l_d>QBm_X?*sbKJu5RgSN5+{)3!v6kcO9Jg`Y&T$9FH#qL(xQpX% zj&E|@!*MUiw>ZAdaUaJzj(_9$4##&n{+;7{z)-mToUZ41faCidZBpa<_kRNiaOdP3 z-0MKk8wAC+NcOqVZ*?TA5l;(8BH0SL`dS3rDObVYC|A4~#rC)2#U&ZNgR;gBOjbdxmA&l))D>g^7i)zKbD0WR< zFJtW8!KxL}?A%~geH1%#hxf&BcI}SI`y$!lA?g(otZm3XIf$J@)#t<6H$zqMpC6`Z zLp{Rf??$nm;quK_u z!(${70+0JV>UA3CZb9*lz@8UE&Iznu$e@?>F4M~_uoZ%$QD83$A?F45VaGe~TrY#j zjtHtsk)0ND5r0FJ5W;G}@)|+DQxx_IUVB90LxFVBIZ=L5WNo5+g-p06deKkg*~|B4 zC>Ty&FS14f{eyD7SU;?h6!);f!zkYbAvT2a%JkkW3MK0Gls9=B1;rXsFbkQC9TXK` zqBfNEH=^8(cMhdUdG=yx8l2Y(8Eca&Wou(ERte@;%KEMLVw+{xz%*O9W(CB`ldVQm z&^PXit)A>d_kD_NywnaQpmRDL9F;ACOJX2S+AU~@(Vsvp%`K-JmeObgJJ0h_1Gl!dOiXjn3Dl{%qtMlqt~&$=FzkzZO| zJ|mwJgxsGPQ5HIEbe@d|>Kk88F4lor!;*?;m6W17s6jed)Ip;xby}8QSM0&m>W)J7 z9VMVGYAR!x+Fr)>5Gfrsc67Qn9SzT8)62n|T(F(#!T<1_vN>$>q;Yo@W;ytai=^l8MeyQzuEb9>|6OMihmY#a)-|PR-Z(Y*wu-+4d-<(WmY^$2FiNN|`*qMNeL5wW| zt{wur0k9C-?L%Nq81~6PFN|S-Xq0VtAwBT$aQp@@@ZGzS28ep$nm_<6k$kM196!P7 z-M!7MAm|_L+$RH-h3`V|#nLPgv zCVW1G3HlJ`zCN6}=f*JkiV;lNI1-H?#biOahXsCQpgAbr^|(0T;(&_-E)KXj;NpOb z11=7@IN;)divunWxH#bAfQthz4!Ahr;(&_-E)KXj;NpOb11=7@IN;)divunWxH#bA zfQthz4!Ahr;(&_-E)KXj;Nrml8yxU9;C>$-Z(LBeA>A<~8n+@QB%G)G+d#N%y<;W&b0EJs|;#N%zC`>TX;9C1Zd zqW{E^?zPhUdpX|6(ROv!+o0ie0>?y-NgPLWOy)R-<5-R<9LI4SFCjD7!1b?-11=7@ zIN;)divunWxH#bAfQthz4!Ahr;(&_-E)KXj;NpOb11=7@IN;)divunWxH#bAfQthz z4!Ahr;(&_-E)KXj;NpOb11=7@IN;)divunWxH#bAfQthz4!Ahr;(&_-E)KXj;NpOb z11=7@IN;)divunWxH#bAfQthz4!Ahr;=u3WfEtk}!!5#(089V^Iv(5K|Jz90-`YF) zHTX3ELq}98oq!(@5T1oQ4W|~EfG-CL_}HXS4=0LD7!CIyxH&Q=c*&XYDjYtTBGWmea*~q~7Kcxb73XB((^?de95WqX%Sz6f z<(N|}MJUXRD=*8_PS488(~iQBj4( zGw^M#ywcL*(xm9*$rEzyZ=wtHie~8ZZLda0T~89LE1Hv4sLR!6mClg9#pdubIx;(p zUX*6d*B0f?)!On%x}Li02o5>iK7V4fO7I=FTxx@jM}LIbv03>5FqQl1Pi+mSRty>{!bYIr{k} zdC>SBO59d%Y!1H4SX3MpU8cteB60hhXA>Hj9q%yG^Rh~l9aTh} zO}St)y2;S9i$=OWPM=pgOIMVo&yzCG7?-Z^dg`FA%E(CNx;}=NF#T5to78(ZBr5)0 ziR^8$eU;-wH#f*kJzTB!SWm>G(qM3G9-mILG`4Y}5o`YNw zfbk0~kYdsFEP3ctvyulrj-?pNm_{S13no%%S4^CiTWA~JMRR9ik*{U=o-uut^xbm& zb9e^6qnb+}pUq3u=_jK(5ReCH6`4Pa^r5uajkSHSb(}6WJ=$?etHY8uS*K%4=?lJ{ zGw3RD1OA;+?XAEhW0ba zYFcV^b_uH%W?}I=FE6JY(vuHH=9RLOf{g9Mubzoo_MI>*OIO5}Q<8liD`M7*Q=b

D#hIZ{hTs!8W>u)BbnZ=sWR4Y!ttQ)0LcF zgdaGg^fjEOZ`2Zf2Y#@P;y2=l&xpqMNusCX2hfOa;xv8BmS}(cup7mHm(%xhx{=d` zoL1n6(Fsp36J6^>55^D2QG5eW{{W}yo4S;LJ*Vm0x8(f4wiz6(tBBu>-!fr-xMG<_$S=yFce_kxM8jEcbw?+PV}8Nz9LK8lj%fnbfW+5M0>+@ z<5-_$CwhSs{e}}=??fMUqSf6x=ReAce$0u6xu;WoRyolhIMFAZ=%1bFp^DD=-Qz^3 zInjkq^eQKMzY~4YiH=uxE>G`7o1EwqPV^8=A&&hq+lhYCiQeQyo1EyKPV^^E^#5_9 ze{`bdn2#LmD;Gx5;&4Y;h5S*Ro@G>rfpZ!w|ZJ?H-LqjnHc8>{N^6G;N8# zwCp+~rQ=*2`D7K%z+qHjp|;~BRN9?(K2#xIcIMVCUM9l*q8RNs@DqnH(J?rcnpIMW z`$N*z5^Y&w@my_5mOg*ncw7*X;!FmNE}4|kiT}4%aoU)*eI5hk3 zxw;b_sntqna{Ts@?FN$*gQszw@tcuvb|b3OxZ#noa~x$ayQ_Op*X!VTVjczdmzChe zkc44+Ug@~o?^KMQj$U;VRr@jAyv{mq9`*n4VVFH19OpS7$lX>x`JMC0zuo%WKGPg{ zc3Umzf)kJeei_a2W(SQkT=T(^Z&qnm4n!W8E@qd*K=hkxG`W-3EatZj>{@fAcJc)D zyJJyNu_%ge;^F8S*HuvJb=m7$Q%g4vqvCkpq?UB3p>+AsUQVp7ob*vz{7;IyCeco!3dD+O{`Zw-#9z!U$oytQhap|F? zw7e*frR=;sU6t_qi|>7JowDj*$32G^B^CTDOl$aMY(zkL*n6J`)unA%754SdiY0GO z*)_^~tln>;*SAv#DU_F9J**wsoc_Ys^Vj`7IrO{dpW5(*DyYQzRo%8@FMb-P4|{R# zmW$`V9;x1&dSC0`r}bDoId*g8wSeT&JAD2&)$B#IFK~rVecKm8J}UTMV-9vy literal 0 HcmV?d00001 From 3aa5e0a99d8b0cd9c8b1a6615013dd46cd4028c6 Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Wed, 1 Apr 2026 16:31:47 +0800 Subject: [PATCH 4/9] update --- .mvn/extensions.xml | 33 - cpp/src/encoding/CMakeLists.txt | 5 +- cpp/src/encoding/bitpacking/int64_packer_n.h | 1126 +++++++++++ .../optimal/sprintz_optimal_pack_size.h | 185 ++ cpp/src/encoding/sprintz_opt_int64_codec.h | 482 +++++ .../encoding/sprintz_opt_int64_codec_test.cc | 122 ++ .../sprintz_optimal_pack_size_test.cc | 80 + ...ntz_packsize8_vs_optimal_benchmark_test.cc | 651 +++++++ .../tsfile/common/conf/TSFileConfig.java | 31 + .../encoding/encoder/LongSprintzEncoder.java | 139 +- .../optimal/SprintzOptimalPackSize.java | 82 +- .../write/writer/ChunkBodyPagedIoWriter.java | 69 + .../tsfile/write/writer/TsFileIOWriter.java | 21 +- .../encoding/AllNo8PacksizeOptimal.java | 1730 +++++++---------- .../tsfile/encoding/ByteArrayTsFileInput.java | 120 ++ .../apache/tsfile/encoding/CuSZpCpuTest.java | 284 ++- .../apache/tsfile/encoding/HBPIndexLong.java | 56 +- .../tsfile/encoding/HBPIndexLongTest.java | 42 +- .../tsfile/encoding/MemoryTsFileOutput.java | 123 ++ .../encoding/TsFilePerPageDiskIoHelper.java | 221 +++ 20 files changed, 4382 insertions(+), 1220 deletions(-) delete mode 100644 .mvn/extensions.xml create mode 100644 cpp/src/encoding/bitpacking/int64_packer_n.h create mode 100644 cpp/src/encoding/optimal/sprintz_optimal_pack_size.h create mode 100644 cpp/src/encoding/sprintz_opt_int64_codec.h create mode 100644 cpp/test/encoding/sprintz_opt_int64_codec_test.cc create mode 100644 cpp/test/encoding/sprintz_optimal_pack_size_test.cc create mode 100644 cpp/test/encoding/sprintz_packsize8_vs_optimal_benchmark_test.cc create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/write/writer/ChunkBodyPagedIoWriter.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/ByteArrayTsFileInput.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/MemoryTsFileOutput.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/encoding/TsFilePerPageDiskIoHelper.java diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml deleted file mode 100644 index 1dd32d62d..000000000 --- a/.mvn/extensions.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - com.gradle - develocity-maven-extension - 1.23.1 - - - com.gradle - common-custom-user-data-maven-extension - 2.0.1 - - diff --git a/cpp/src/encoding/CMakeLists.txt b/cpp/src/encoding/CMakeLists.txt index 45c1f0bdd..da6d606bd 100644 --- a/cpp/src/encoding/CMakeLists.txt +++ b/cpp/src/encoding/CMakeLists.txt @@ -22,5 +22,8 @@ message("Running in src/encoding directory") add_custom_target( encoding_obj ALL ) -file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +file(GLOB_RECURSE HEADERS + "${CMAKE_CURRENT_SOURCE_DIR}/*.h" + "${CMAKE_CURRENT_SOURCE_DIR}/*.hpp" +) copy_to_dir(${HEADERS} "encoding_obj") \ No newline at end of file diff --git a/cpp/src/encoding/bitpacking/int64_packer_n.h b/cpp/src/encoding/bitpacking/int64_packer_n.h new file mode 100644 index 000000000..2dc81ab9a --- /dev/null +++ b/cpp/src/encoding/bitpacking/int64_packer_n.h @@ -0,0 +1,1126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_BITPACKING_INT64_PACKER_N_H +#define ENCODING_BITPACKING_INT64_PACKER_N_H + +#include +#include + +#if defined(__ARM_NEON) +#include +#endif + +namespace storage { +namespace bitpacking { + +// Bit-packer that supports variable N (1..32). Ported from Java LongPacker.packNValues/unpackNValues. +class Int64PackerN { + public: + explicit Int64PackerN(int width_bits, bool enable_simd = true) + : width_(width_bits), enable_simd_(enable_simd) {} + + void set_width(int width_bits) { width_ = width_bits; } + void set_enable_simd(bool enable) { enable_simd_ = enable; } + + void pack_n_values(const int64_t *values, int offset, int n, uint8_t *buf) const { + // Width-dispatched specialized kernels (reduces branches; allows compiler to unroll). + // Fallback to generic for uncommon widths. + switch (width_) { + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + case 22: + case 23: + case 24: + case 25: + case 26: + case 27: + case 28: + case 29: + case 30: + case 31: + case 32: + case 33: + pack_n_values_w(values + offset, n, buf, width_, enable_simd_); + return; + default: + pack_n_values_generic(values, offset, n, buf); + return; + } + } + + void unpack_n_values(const uint8_t *buf, int offset, int n, int64_t *values) const { + switch (width_) { + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + case 22: + case 23: + case 24: + case 25: + case 26: + case 27: + case 28: + case 29: + case 30: + case 31: + case 32: + case 33: + unpack_n_values_w(buf + offset, n, values, width_, enable_simd_); + return; + default: + unpack_n_values_generic(buf, offset, n, values); + return; + } + } + + private: + static FORCE_INLINE uint64_t mask_w_u64(int w) { + return (w >= 64) ? ~0ULL : ((1ULL << w) - 1ULL); + } + + static FORCE_INLINE void pack_n_values_w(const int64_t *values, int n, uint8_t *buf, int w, bool enable_simd) { + // Fast paths for byte-aligned widths. These are also the most common widths after bit-width selection. + // Encoding is big-endian within each value (matches the bit-buffer emission order). + if (enable_simd && w == 6) { + pack_n_values_6(values, n, buf); + return; + } + if (enable_simd && w == 7) { + pack_n_values_7(values, n, buf); + return; + } + if (enable_simd && w == 8) { + pack_n_values_8(values, n, buf); + return; + } + if (enable_simd && w == 9) { + pack_n_values_9(values, n, buf); + return; + } + if (enable_simd && w == 10) { + pack_n_values_10(values, n, buf); + return; + } + if (enable_simd && w == 11) { + pack_n_values_11(values, n, buf); + return; + } + if (enable_simd && w == 12) { + pack_n_values_12(values, n, buf); + return; + } + if (enable_simd && w == 16) { + pack_n_values_16(values, n, buf); + return; + } + if (enable_simd && w == 24) { + pack_n_values_24(values, n, buf); + return; + } + if (enable_simd && w == 32) { + pack_n_values_32(values, n, buf); + return; + } + if (enable_simd && w == 33) { + pack_n_values_33(values, n, buf); + return; + } +#if defined(__ARM_NEON) + if (enable_simd && w > 0 && w < 32) { + pack_n_values_neon_generic(values, n, buf, w); + return; + } +#endif + // Pack using a bit-buffer; emits big-endian bytes for each 64-bit chunk (matches Java logic). + // This is still scalar but specialized by w, and has a tighter loop than the generic. + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i]; + bitbuf = (bitbuf << w) | (v & ((w == 64) ? ~0ULL : ((1ULL << w) - 1ULL))); + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + if (bits > 0 && out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } + // Caller guarantees buffer has enough space; no need to pad further. + } + + static FORCE_INLINE void unpack_n_values_w(const uint8_t *buf, int n, int64_t *values, int w, bool enable_simd) { + if (enable_simd && w == 8) { + unpack_n_values_8(buf, n, values); + return; + } + if (enable_simd && w == 16) { + unpack_n_values_16(buf, n, values); + return; + } + if (enable_simd && w == 24) { + unpack_n_values_24(buf, n, values); + return; + } + if (enable_simd && w == 32) { + unpack_n_values_32(buf, n, values); + return; + } +#if defined(__ARM_NEON) + if (enable_simd && w > 0 && w < 32) { + unpack_n_values_neon_generic(buf, n, values, w); + return; + } +#endif + int byte_idx = 0; + uint64_t bitbuf = 0; + int bits = 0; + const uint64_t mask = mask_w_u64(w); + for (int i = 0; i < n; i++) { + while (bits < w) { + bitbuf = (bitbuf << 8) | (uint64_t)(buf[byte_idx++] & 0xFF); + bits += 8; + } + bits -= w; + values[i] = (int64_t)((bitbuf >> bits) & mask); + } + } + + static FORCE_INLINE void pack_n_values_8(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + // Process 16 values per iteration. We still need to narrow from int64 -> u8. + for (; i + 15 < n; i += 16) { + // Load 16x64-bit as 8x uint64x2_t + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint64x2_t v4 = vld1q_u64(reinterpret_cast(values + i + 8)); + uint64x2_t v5 = vld1q_u64(reinterpret_cast(values + i + 10)); + uint64x2_t v6 = vld1q_u64(reinterpret_cast(values + i + 12)); + uint64x2_t v7 = vld1q_u64(reinterpret_cast(values + i + 14)); + + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint32x2_t n4 = vmovn_u64(v4); + uint32x2_t n5 = vmovn_u64(v5); + uint32x2_t n6 = vmovn_u64(v6); + uint32x2_t n7 = vmovn_u64(v7); + + uint16x4_t m0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t m1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x4_t m2 = vmovn_u32(vcombine_u32(n4, n5)); + uint16x4_t m3 = vmovn_u32(vcombine_u32(n6, n7)); + + uint8x8_t b01 = vmovn_u16(vcombine_u16(m0, m1)); // 8 bytes + uint8x8_t b23 = vmovn_u16(vcombine_u16(m2, m3)); // 8 bytes + + vst1_u8(buf + i, b01); + vst1_u8(buf + i + 8, b23); + } + for (; i < n; i++) { + buf[i] = (uint8_t)(values[i] & 0xFF); + } +#else + for (int i = 0; i < n; i++) { + buf[i] = (uint8_t)(values[i] & 0xFF); + } +#endif + } + + static FORCE_INLINE void unpack_n_values_8(const uint8_t *buf, int n, int64_t *values) { + for (int i = 0; i < n; i++) { + values[i] = (int64_t)(buf[i] & 0xFF); + } + } + + static FORCE_INLINE void pack_n_values_16(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + // 8 values -> 16 bytes + for (; i + 7 < n; i += 8) { + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint16x4_t u0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t u1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x8_t u = vcombine_u16(u0, u1); + + // big-endian bytes per u16: hi then lo + uint8x16_t lo = vreinterpretq_u8_u16(u); + // swap bytes within each 16-bit lane + uint8x16_t be = vrev16q_u8(lo); + vst1q_u8(buf + (i * 2), be); + } + for (; i < n; i++) { + uint16_t v = (uint16_t)(values[i] & 0xFFFF); + buf[i * 2] = (uint8_t)(v >> 8); + buf[i * 2 + 1] = (uint8_t)(v & 0xFF); + } +#else + for (int i = 0; i < n; i++) { + uint16_t v = (uint16_t)(values[i] & 0xFFFF); + buf[i * 2] = (uint8_t)(v >> 8); + buf[i * 2 + 1] = (uint8_t)(v & 0xFF); + } +#endif + } + + static FORCE_INLINE void unpack_n_values_16(const uint8_t *buf, int n, int64_t *values) { + for (int i = 0; i < n; i++) { + uint16_t v = ((uint16_t)(buf[i * 2] & 0xFF) << 8) | (uint16_t)(buf[i * 2 + 1] & 0xFF); + values[i] = (int64_t)v; + } + } + + static FORCE_INLINE void pack_n_values_12(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint16x8_t vmask = vdupq_n_u16((uint16_t)0x0FFF); + uint16_t tmp16[8]; + for (; i + 7 < n; i += 8) { // 8 values -> 12 bytes + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint16x4_t u0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t u1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x8_t u = vcombine_u16(u0, u1); + u = vandq_u16(u, vmask); + vst1q_u16(tmp16, u); + // Big-endian 12-bit stream: per pair -> 3 bytes. + for (int k = 0; k < 8; k += 2) { + uint16_t a = tmp16[k]; + uint16_t b = tmp16[k + 1]; + buf[out++] = (uint8_t)(a >> 4); + buf[out++] = (uint8_t)(((a & 0x0F) << 4) | (uint8_t)(b >> 8)); + buf[out++] = (uint8_t)(b & 0xFF); + } + } + // tail: scalar bit-buffer (same semantics) + const int w = 12; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + // If we already emitted full chunks, keep out aligned (should be <= byte_limit). + // out already equals (i/8)*12 bytes here. + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x0FFFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + if (bits > 0 && out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } +#else + // Scalar (same as generic, but unrolled by pairs for w=12) + int out = 0; + int i = 0; + for (; i + 1 < n; i += 2) { + uint16_t a = (uint16_t)((uint64_t)values[i] & 0x0FFFULL); + uint16_t b = (uint16_t)((uint64_t)values[i + 1] & 0x0FFFULL); + buf[out++] = (uint8_t)(a >> 4); + buf[out++] = (uint8_t)(((a & 0x0F) << 4) | (uint8_t)(b >> 8)); + buf[out++] = (uint8_t)(b & 0xFF); + } + if (i < n) { + uint16_t a = (uint16_t)((uint64_t)values[i] & 0x0FFFULL); + buf[out++] = (uint8_t)(a >> 4); + buf[out++] = (uint8_t)((a & 0x0F) << 4); + } +#endif + } + + static FORCE_INLINE void pack_n_values_6(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint8x16_t vmask = vdupq_n_u8((uint8_t)0x3F); + uint8_t tmp8[16]; + for (; i + 15 < n; i += 16) { // 16 values -> 12 bytes + // Load 16x64 -> narrow to 16x8 (low bits), then pack 6-bit stream. + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint64x2_t v4 = vld1q_u64(reinterpret_cast(values + i + 8)); + uint64x2_t v5 = vld1q_u64(reinterpret_cast(values + i + 10)); + uint64x2_t v6 = vld1q_u64(reinterpret_cast(values + i + 12)); + uint64x2_t v7 = vld1q_u64(reinterpret_cast(values + i + 14)); + + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint32x2_t n4 = vmovn_u64(v4); + uint32x2_t n5 = vmovn_u64(v5); + uint32x2_t n6 = vmovn_u64(v6); + uint32x2_t n7 = vmovn_u64(v7); + + uint16x4_t m0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t m1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x4_t m2 = vmovn_u32(vcombine_u32(n4, n5)); + uint16x4_t m3 = vmovn_u32(vcombine_u32(n6, n7)); + + uint8x8_t b01 = vmovn_u16(vcombine_u16(m0, m1)); + uint8x8_t b23 = vmovn_u16(vcombine_u16(m2, m3)); + uint8x16_t b = vcombine_u8(b01, b23); + b = vandq_u8(b, vmask); + vst1q_u8(tmp8, b); + + // Pack 16x6 bits => 12 bytes (big-endian bitstream): + // For each group of 4 values (24 bits) -> 3 bytes. + for (int k = 0; k < 16; k += 4) { + uint32_t a = (uint32_t)tmp8[k + 0]; + uint32_t b2 = (uint32_t)tmp8[k + 1]; + uint32_t c = (uint32_t)tmp8[k + 2]; + uint32_t d = (uint32_t)tmp8[k + 3]; + uint32_t bits24 = (a << 18) | (b2 << 12) | (c << 6) | d; + buf[out++] = (uint8_t)(bits24 >> 16); + buf[out++] = (uint8_t)(bits24 >> 8); + buf[out++] = (uint8_t)(bits24); + } + } + // tail: scalar bit-buffer + const int w = 6; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x3FULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + if (bits > 0 && out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } +#else + // Scalar grouped packing: 4 values -> 3 bytes + int out = 0; + int i = 0; + for (; i + 3 < n; i += 4) { + uint32_t a = (uint32_t)((uint64_t)values[i + 0] & 0x3FULL); + uint32_t b = (uint32_t)((uint64_t)values[i + 1] & 0x3FULL); + uint32_t c = (uint32_t)((uint64_t)values[i + 2] & 0x3FULL); + uint32_t d = (uint32_t)((uint64_t)values[i + 3] & 0x3FULL); + uint32_t bits24 = (a << 18) | (b << 12) | (c << 6) | d; + buf[out++] = (uint8_t)(bits24 >> 16); + buf[out++] = (uint8_t)(bits24 >> 8); + buf[out++] = (uint8_t)(bits24); + } + if (i < n) { + const int w = 6; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x3FULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } +#endif + } + + static FORCE_INLINE void pack_n_values_7(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint8x16_t vmask = vdupq_n_u8((uint8_t)0x7F); + uint8_t tmp8[16]; + for (; i + 15 < n; i += 16) { // 16 values -> 14 bytes + // Load 16x64 -> narrow to 16x8 (low bits), then pack 7-bit stream. + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint64x2_t v4 = vld1q_u64(reinterpret_cast(values + i + 8)); + uint64x2_t v5 = vld1q_u64(reinterpret_cast(values + i + 10)); + uint64x2_t v6 = vld1q_u64(reinterpret_cast(values + i + 12)); + uint64x2_t v7 = vld1q_u64(reinterpret_cast(values + i + 14)); + + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint32x2_t n4 = vmovn_u64(v4); + uint32x2_t n5 = vmovn_u64(v5); + uint32x2_t n6 = vmovn_u64(v6); + uint32x2_t n7 = vmovn_u64(v7); + + uint16x4_t m0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t m1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x4_t m2 = vmovn_u32(vcombine_u32(n4, n5)); + uint16x4_t m3 = vmovn_u32(vcombine_u32(n6, n7)); + + uint8x8_t b01 = vmovn_u16(vcombine_u16(m0, m1)); + uint8x8_t b23 = vmovn_u16(vcombine_u16(m2, m3)); + uint8x16_t b = vcombine_u8(b01, b23); + b = vandq_u8(b, vmask); + vst1q_u8(tmp8, b); + + // Pack 8x7 bits => 7 bytes, twice per 16 values. + for (int base = 0; base < 16; base += 8) { + uint64_t bitbuf = 0; + int bits = 0; + for (int k = 0; k < 8; k++) { + bitbuf = (bitbuf << 7) | (uint64_t)tmp8[base + k]; + bits += 7; + while (bits >= 8) { + bits -= 8; + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + // bits should be 0 here (56 bits -> 7 bytes) + } + } + // tail: scalar bit-buffer + const int w = 7; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x7FULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + if (bits > 0 && out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } +#else + // Scalar bit-buffer + const int w = 7; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x7FULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#endif + } + + static FORCE_INLINE void pack_n_values_9(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint16x8_t vmask = vdupq_n_u16((uint16_t)0x01FF); + uint16_t tmp16[8]; + for (; i + 7 < n; i += 8) { // 8 values -> 9 bytes + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint16x4_t u0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t u1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x8_t u = vcombine_u16(u0, u1); + u = vandq_u16(u, vmask); + vst1q_u16(tmp16, u); + + uint64_t bitbuf = 0; + int bits = 0; + for (int k = 0; k < 8; k++) { + bitbuf = (bitbuf << 9) | (uint64_t)tmp16[k]; + bits += 9; + while (bits >= 8) { + bits -= 8; + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + // 72 bits -> 9 bytes, bits ends at 0. + } + // tail: scalar bit-buffer + const int w = 9; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x01FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#else + // Scalar bit-buffer + const int w = 9; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x01FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#endif + } + + static FORCE_INLINE void pack_n_values_10(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint16x8_t vmask = vdupq_n_u16((uint16_t)0x03FF); + uint16_t tmp16[8]; + for (; i + 7 < n; i += 8) { // 8 values -> 10 bytes + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint16x4_t u0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t u1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x8_t u = vcombine_u16(u0, u1); + u = vandq_u16(u, vmask); + vst1q_u16(tmp16, u); + + uint64_t bitbuf = 0; + int bits = 0; + for (int k = 0; k < 8; k++) { + bitbuf = (bitbuf << 10) | (uint64_t)tmp16[k]; + bits += 10; + while (bits >= 8) { + bits -= 8; + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + // 80 bits -> 10 bytes + } + // tail: scalar bit-buffer + const int w = 10; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x03FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#else + const int w = 10; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x03FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#endif + } + + static FORCE_INLINE void pack_n_values_11(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + int out = 0; + const uint16x8_t vmask = vdupq_n_u16((uint16_t)0x07FF); + uint16_t tmp16[8]; + for (; i + 7 < n; i += 8) { // 8 values -> 11 bytes + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint64x2_t v2 = vld1q_u64(reinterpret_cast(values + i + 4)); + uint64x2_t v3 = vld1q_u64(reinterpret_cast(values + i + 6)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x2_t n2 = vmovn_u64(v2); + uint32x2_t n3 = vmovn_u64(v3); + uint16x4_t u0 = vmovn_u32(vcombine_u32(n0, n1)); + uint16x4_t u1 = vmovn_u32(vcombine_u32(n2, n3)); + uint16x8_t u = vcombine_u16(u0, u1); + u = vandq_u16(u, vmask); + vst1q_u16(tmp16, u); + + uint64_t bitbuf = 0; + int bits = 0; + for (int k = 0; k < 8; k++) { + bitbuf = (bitbuf << 11) | (uint64_t)tmp16[k]; + bits += 11; + while (bits >= 8) { + bits -= 8; + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + // 88 bits -> 11 bytes + } + // tail: scalar bit-buffer + const int w = 11; + const int byte_limit = (n * w + 7) / 8; + uint64_t bitbuf = 0; + int bits = 0; + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x07FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#else + const int w = 11; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x07FFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#endif + } + + static FORCE_INLINE void pack_n_values_33(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + const int w = 33; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + const uint64x2_t vmask = vdupq_n_u64(0x1FFFFFFFFULL); + uint64_t tmp64[2]; + int i = 0; + for (; i + 1 < n; i += 2) { + uint64x2_t v = vld1q_u64(reinterpret_cast(values + i)); + v = vandq_u64(v, vmask); + vst1q_u64(tmp64, v); + for (int k = 0; k < 2; k++) { + bitbuf = (bitbuf << w) | (tmp64[k] & 0x1FFFFFFFFULL); + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x1FFFFFFFFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#else + const int w = 33; + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)values[i] & 0x1FFFFFFFFULL; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + if (bits > 0 && out < byte_limit) buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); +#endif + } + + static FORCE_INLINE void pack_n_values_24(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + for (; i + 3 < n; i += 4) { // 4 values -> 12 bytes + // Load 4x64-bit (two vectors), narrow to 4x32-bit, then write 3 bytes/value big-endian. + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x4_t x = vcombine_u32(n0, n1); // [x0 x1 x2 x3] as u32 lanes + + // Convert to bytes in big-endian order per u32 lane, then drop the top byte. + uint8x16_t b = vreinterpretq_u8_u32(x); + uint8x16_t be = vrev32q_u8(b); // for each u32: [b3 b2 b1 b0] + + uint8_t tmp[16]; + vst1q_u8(tmp, be); + // tmp layout after vrev32: x0(4B),x1(4B),x2(4B),x3(4B) each big-endian. + // Emit low 24 bits => drop tmp[k*4 + 0], keep [1..3]. + for (int k = 0; k < 4; k++) { + const int o = (i + k) * 3; + buf[o + 0] = tmp[k * 4 + 1]; + buf[o + 1] = tmp[k * 4 + 2]; + buf[o + 2] = tmp[k * 4 + 3]; + } + } + for (; i < n; i++) { + uint32_t v = (uint32_t)(values[i] & 0xFFFFFFu); + const int o = i * 3; + buf[o + 0] = (uint8_t)(v >> 16); + buf[o + 1] = (uint8_t)(v >> 8); + buf[o + 2] = (uint8_t)(v & 0xFF); + } +#else + for (int i = 0; i < n; i++) { + uint32_t v = (uint32_t)(values[i] & 0xFFFFFFu); + const int o = i * 3; + buf[o + 0] = (uint8_t)(v >> 16); + buf[o + 1] = (uint8_t)(v >> 8); + buf[o + 2] = (uint8_t)(v & 0xFF); + } +#endif + } + + static FORCE_INLINE void unpack_n_values_24(const uint8_t *buf, int n, int64_t *values) { + for (int i = 0; i < n; i++) { + const int o = i * 3; + uint32_t v = ((uint32_t)(buf[o + 0] & 0xFF) << 16) | ((uint32_t)(buf[o + 1] & 0xFF) << 8) | + (uint32_t)(buf[o + 2] & 0xFF); + values[i] = (int64_t)v; + } + } + + static FORCE_INLINE void pack_n_values_32(const int64_t *values, int n, uint8_t *buf) { +#if defined(__ARM_NEON) + int i = 0; + for (; i + 3 < n; i += 4) { // 4 values -> 16 bytes + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x4_t x = vcombine_u32(n0, n1); + // Store big-endian per u32. + uint8x16_t b = vreinterpretq_u8_u32(x); + uint8x16_t be = vrev32q_u8(b); + vst1q_u8(buf + (i * 4), be); + } + for (; i < n; i++) { + uint32_t v = (uint32_t)(values[i] & 0xFFFFFFFFu); + const int o = i * 4; + buf[o + 0] = (uint8_t)(v >> 24); + buf[o + 1] = (uint8_t)(v >> 16); + buf[o + 2] = (uint8_t)(v >> 8); + buf[o + 3] = (uint8_t)(v & 0xFF); + } +#else + for (int i = 0; i < n; i++) { + uint32_t v = (uint32_t)(values[i] & 0xFFFFFFFFu); + const int o = i * 4; + buf[o + 0] = (uint8_t)(v >> 24); + buf[o + 1] = (uint8_t)(v >> 16); + buf[o + 2] = (uint8_t)(v >> 8); + buf[o + 3] = (uint8_t)(v & 0xFF); + } +#endif + } + + static FORCE_INLINE void unpack_n_values_32(const uint8_t *buf, int n, int64_t *values) { + for (int i = 0; i < n; i++) { + const int o = i * 4; + uint32_t v = ((uint32_t)(buf[o + 0] & 0xFF) << 24) | ((uint32_t)(buf[o + 1] & 0xFF) << 16) | + ((uint32_t)(buf[o + 2] & 0xFF) << 8) | (uint32_t)(buf[o + 3] & 0xFF); + values[i] = (int64_t)v; + } + } + +#if defined(__ARM_NEON) + // Generic NEON-assisted bit-buffer pack/unpack for widths 1..31. + // Semantics MUST match Java LongPacker.packNValues/unpackNValues (big-endian bitstream). + // + // Note: This is not a byte-aligned memcpy; it uses NEON to speed up value ingest/masking + // and (for unpack) batch stores. The bitstream assembly still uses a scalar bit-buffer, + // because big-endian arbitrary-bit packing is inherently a cross-lane operation. + static FORCE_INLINE void pack_n_values_neon_generic(const int64_t *values, int n, uint8_t *buf, int w) { + const int byte_limit = (n * w + 7) / 8; + int out = 0; + uint64_t bitbuf = 0; + int bits = 0; + + const uint32_t mask32 = (w == 32) ? 0xFFFFFFFFu : ((1u << w) - 1u); + const uint32x4_t vmask = vdupq_n_u32(mask32); + + int i = 0; + uint32_t tmp[4]; + for (; i + 3 < n; i += 4) { + uint64x2_t v0 = vld1q_u64(reinterpret_cast(values + i + 0)); + uint64x2_t v1 = vld1q_u64(reinterpret_cast(values + i + 2)); + uint32x2_t n0 = vmovn_u64(v0); + uint32x2_t n1 = vmovn_u64(v1); + uint32x4_t x = vcombine_u32(n0, n1); + x = vandq_u32(x, vmask); + vst1q_u32(tmp, x); + + // Append 4 masked values to big-endian bitstream. + for (int k = 0; k < 4; k++) { + bitbuf = (bitbuf << w) | (uint64_t)(tmp[k] & mask32); + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + } + for (; i < n; i++) { + uint64_t v = (uint64_t)values[i] & (uint64_t)mask32; + bitbuf = (bitbuf << w) | v; + bits += w; + while (bits >= 8) { + bits -= 8; + if (out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf >> bits) & 0xFF); + } + } + } + if (bits > 0 && out < byte_limit) { + buf[out++] = (uint8_t)((bitbuf << (8 - bits)) & 0xFF); + } + } + + static FORCE_INLINE void unpack_n_values_neon_generic(const uint8_t *buf, int n, int64_t *values, int w) { + int byte_idx = 0; + uint64_t bitbuf = 0; + int bits = 0; + const uint64_t mask = mask_w_u64(w); + + int i = 0; + for (; i + 3 < n; i += 4) { + uint32_t out4[4]; + for (int k = 0; k < 4; k++) { + while (bits < w) { + bitbuf = (bitbuf << 8) | (uint64_t)(buf[byte_idx++] & 0xFF); + bits += 8; + } + bits -= w; + out4[k] = (uint32_t)((bitbuf >> bits) & mask); + } + uint32x4_t v = vld1q_u32(out4); + // Widen to u64 lanes then store as int64_t scalars. + uint32x2_t lo = vget_low_u32(v); + uint32x2_t hi = vget_high_u32(v); + uint64x2_t vlo = vmovl_u32(lo); + uint64x2_t vhi = vmovl_u32(hi); + uint64_t tmp64[4]; + vst1q_u64(tmp64 + 0, vlo); + vst1q_u64(tmp64 + 2, vhi); + values[i + 0] = (int64_t)tmp64[0]; + values[i + 1] = (int64_t)tmp64[1]; + values[i + 2] = (int64_t)tmp64[2]; + values[i + 3] = (int64_t)tmp64[3]; + } + for (; i < n; i++) { + while (bits < w) { + bitbuf = (bitbuf << 8) | (uint64_t)(buf[byte_idx++] & 0xFF); + bits += 8; + } + bits -= w; + values[i] = (int64_t)((bitbuf >> bits) & mask); + } + } +#endif + + void pack_n_values_generic(const int64_t *values, int offset, int n, uint8_t *buf) const { + int buf_idx = 0; + int value_idx = offset; + int left_bit = 0; + int byte_limit = (n * width_ + 7) / 8; + + while (value_idx < n + offset && buf_idx < byte_limit) { + uint64_t buffer = 0; + int left_size = 64; + + if (left_bit > 0) { + buffer |= ((uint64_t)values[value_idx] << (64 - left_bit)); + left_size -= left_bit; + left_bit = 0; + value_idx++; + } + + while (left_size >= width_ && value_idx < n + offset) { + buffer |= ((uint64_t)values[value_idx] << (left_size - width_)); + left_size -= width_; + value_idx++; + } + if (left_size > 0 && value_idx < n + offset) { + buffer |= ((uint64_t)values[value_idx] >> (width_ - left_size)); + left_bit = width_ - left_size; + } + + for (int j = 0; j < 8 && buf_idx < byte_limit; j++) { + buf[buf_idx] = (uint8_t)((buffer >> ((8 - j - 1) * 8)) & 0xFF); + buf_idx++; + } + } + } + + void unpack_n_values_generic(const uint8_t *buf, int offset, int n, int64_t *values) const { + int byte_idx = offset; + int value_idx = 0; + int left_bits = 8; + int total_bits = 0; + + while (value_idx < n) { + uint64_t v = 0; + total_bits = 0; + while (total_bits < width_) { + if (width_ - total_bits >= left_bits) { + v <<= left_bits; + v |= ((uint64_t)((1ULL << left_bits) - 1ULL) & + (uint64_t)(buf[byte_idx] & 0xFF)); + total_bits += left_bits; + byte_idx++; + left_bits = 8; + } else { + int t = width_ - total_bits; + v <<= t; + v |= ((uint64_t)((((1ULL << left_bits) - 1ULL) & + (uint64_t)(buf[byte_idx] & 0xFF)) >> + (left_bits - t))); + left_bits -= t; + total_bits += t; + } + } + values[value_idx] = (int64_t)v; + value_idx++; + } + } + + int width_; + bool enable_simd_; +}; + +} // namespace bitpacking +} // namespace storage + +#endif // ENCODING_BITPACKING_INT64_PACKER_N_H + diff --git a/cpp/src/encoding/optimal/sprintz_optimal_pack_size.h b/cpp/src/encoding/optimal/sprintz_optimal_pack_size.h new file mode 100644 index 000000000..12e351325 --- /dev/null +++ b/cpp/src/encoding/optimal/sprintz_optimal_pack_size.h @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_OPTIMAL_SPRINTZ_OPTIMAL_PACK_SIZE_H +#define ENCODING_OPTIMAL_SPRINTZ_OPTIMAL_PACK_SIZE_H + +#include +#include +#include + +#if defined(__ARM_NEON) +#include +#endif + +namespace storage { +namespace optimal { + +// Ported from Java: org.apache.tsfile.encoding.optimal.SprintzOptimalPackSize +// Cost model: sum(pack_len * max_bitwidth_in_pack) + num_packs * bitsPerBlockOverhead. +class SprintzOptimalPackSize { + public: + static int find_optimal_pack_size(const int64_t *values, int n, + std::vector *bw_scratch = nullptr, + bool enable_simd = false) { + if (n <= 0) { + return 1; + } + if (n < 8) { + return std::max(1, n); + } + + std::vector local_bw; + std::vector *bw = bw_scratch; + if (bw == nullptr) { + local_bw.resize(n); + bw = &local_bw; + } else if ((int)bw->size() < n) { + bw->resize(n); + } + + // Compute per-value bitwidths; NEON best-effort on arm64. + if (enable_simd) { +#if defined(__ARM_NEON) + int i = 0; + // clamp to >=1 so bitwidth>=1 + const uint64x2_t vone = vdupq_n_u64(1); + for (; i + 1 < n; i += 2) { + uint64x2_t v = vld1q_u64(reinterpret_cast(values + i)); + // vmax(v,1) without relying on vmaxq_u64 + uint64x2_t mask = vcgtq_u64(v, vone); + v = vbslq_u64(mask, v, vone); + // clz: some toolchains don't expose vclzq_u64; use 32-bit lanes (values here are <= 2^64-1). + uint32x4_t v32 = vreinterpretq_u32_u64(v); + uint32x4_t clz32 = vclzq_u32(v32); + // Each original 64-bit lane uses two 32-bit lanes. + uint64_t lanes[2]; + uint32_t c32[4]; + vst1q_u32(c32, clz32); + // Reconstruct clz64 for each lane: + // If high32 != 0 => clz64 = clz(high32) + // else clz64 = 32 + clz(low32) + auto clz64_from_parts = [](uint32_t high32, uint32_t low32) -> uint64_t { + if (high32 != 0) { + return (uint64_t)__builtin_clz(high32); + } + return 32ULL + (uint64_t)__builtin_clz(low32); + }; + // c32 holds clz of each 32-bit lane, but we need clz of the value parts. + // Compute high/low words. + uint64_t x0 = ((uint64_t)vgetq_lane_u32(v32, 1) << 32) | (uint64_t)vgetq_lane_u32(v32, 0); + uint64_t x1 = ((uint64_t)vgetq_lane_u32(v32, 3) << 32) | (uint64_t)vgetq_lane_u32(v32, 2); + uint32_t x0_hi = (uint32_t)(x0 >> 32), x0_lo = (uint32_t)(x0 & 0xffffffffu); + uint32_t x1_hi = (uint32_t)(x1 >> 32), x1_lo = (uint32_t)(x1 & 0xffffffffu); + lanes[0] = clz64_from_parts(x0_hi, x0_lo); + lanes[1] = clz64_from_parts(x1_hi, x1_lo); + (*bw)[i] = (int)(64 - lanes[0]); + (*bw)[i + 1] = (int)(64 - lanes[1]); + } + for (; i < n; i++) { + uint64_t v = (uint64_t)std::max(1, values[i]); + (*bw)[i] = 64 - __builtin_clzll(v); + } +#else + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)std::max(1, values[i]); + (*bw)[i] = 64 - __builtin_clzll(v); + } +#endif + } else { + for (int i = 0; i < n; i++) { + uint64_t v = (uint64_t)std::max(1, values[i]); + (*bw)[i] = 64 - __builtin_clzll(v); + } + } + + const int bits_per_block_overhead = 80; // 1B packSize + 1B bitWidth + 8B preValue + + int best_pack_size = 1; + int max_pack_size = std::min(32, n); + int64_t best_cost = INT64_MAX; + +#if defined(__ARM_NEON) + // If SIMD enabled, also keep an 8-bit copy of bitwidths for fast vmax reductions. + std::vector bw8_local; + const uint8_t *bw8 = nullptr; + if (enable_simd) { + bw8_local.resize((size_t)n); + for (int i = 0; i < n; i++) { + int v = (*bw)[i]; + if (v < 1) v = 1; + if (v > 255) v = 255; + bw8_local[(size_t)i] = (uint8_t)v; + } + bw8 = bw8_local.data(); + } + auto max_u8_block = [](const uint8_t *p, int len) -> int { + if (len <= 0) return 1; + int i = 0; + uint8x16_t vmax16 = vdupq_n_u8(1); + for (; i + 15 < len; i += 16) { + uint8x16_t v = vld1q_u8(p + i); + vmax16 = vmaxq_u8(vmax16, v); + } + uint8_t m = vmaxvq_u8(vmax16); + for (; i < len; i++) { + if (p[i] > m) m = p[i]; + } + return (int)m; + }; +#endif + + for (int p = 1; p <= max_pack_size; p++) { + int m = (n + p - 1) / p; + int64_t cost = 0; + for (int i = 0; i < m; i++) { + int start = i * p; + int end = std::min(start + p, n); + int max_bw = 1; +#if defined(__ARM_NEON) + if (enable_simd && bw8 != nullptr) { + max_bw = max_u8_block(bw8 + start, end - start); + } else { + for (int j = start; j < end; j++) { + max_bw = std::max(max_bw, (*bw)[j]); + } + } +#else + for (int j = start; j < end; j++) { + max_bw = std::max(max_bw, (*bw)[j]); + } +#endif + cost += (int64_t)(end - start) * (int64_t)max_bw; + } + cost += (int64_t)m * (int64_t)bits_per_block_overhead; + if (cost < best_cost) { + best_cost = cost; + best_pack_size = p; + } + } + + return best_pack_size; + } +}; + +} // namespace optimal +} // namespace storage + +#endif // ENCODING_OPTIMAL_SPRINTZ_OPTIMAL_PACK_SIZE_H + diff --git a/cpp/src/encoding/sprintz_opt_int64_codec.h b/cpp/src/encoding/sprintz_opt_int64_codec.h new file mode 100644 index 000000000..1f25ce924 --- /dev/null +++ b/cpp/src/encoding/sprintz_opt_int64_codec.h @@ -0,0 +1,482 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_SPRINTZ_OPT_INT64_CODEC_H +#define ENCODING_SPRINTZ_OPT_INT64_CODEC_H + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__ARM_NEON) +#include +#endif + +#include "common/allocator/byte_stream.h" +#include "encoding/bitpacking/int64_packer_n.h" +#include "encoding/decoder.h" +#include "encoding/encoder.h" +#include "encoding/fire.h" +#include "encoding/int64_rle_decoder.h" +#include "encoding/int64_rle_encoder.h" +#include "encoding/optimal/sprintz_optimal_pack_size.h" + +namespace storage { + +// A minimal C++ port of Java LongSprintzEncoder/LongSprintzDecoder optimal mode. +// - Predict scheme: "fire" by default (matches C++ Sprintz default), can be set to "delta". +// - Stream starts with [0][1] marker for optimal mode. +// - Each block: [packSize:1B][bitWidth:1B][preValue:8B][packed residuals] +// - Special cases: +// - single value: [0][0][preValue:8B] +// - legacy RLE block (for trailing in legacy mode): not used here. +class SprintzOptInt64Encoder final : public Encoder { + public: + static constexpr bool kNeonEnabled = +#if defined(__ARM_NEON) + true; +#else + false; +#endif + + explicit SprintzOptInt64Encoder(bool use_optimal_pack_size, int fixed_pack_size = 8, + const std::string &predict_method = "delta", + int optimal_chunk_min_size = 1024, + bool enable_simd = false) + : use_optimal_(use_optimal_pack_size), + fixed_pack_size_(std::max(1, std::min(32, fixed_pack_size))), + predict_method_(predict_method), + optimal_chunk_min_size_(std::max(2, optimal_chunk_min_size)), + enable_simd_(enable_simd), + fire_pred_(3), + packer_n_(1, enable_simd) { + reset(); + } + + void reset() override { + values_.clear(); + chunk_.clear(); + marker_written_ = false; + bw_scratch_.clear(); + bw_hist_.fill(0); + bw_hist_total_blocks_ = 0; + collect_bw_stats_ = false; + bw_stats_topk_ = 0; + if (use_optimal_) { + if (const char *env = std::getenv("TSFILE_OPT_BW_TOPK")) { + int k = atoi(env); + if (k > 0) { + collect_bw_stats_ = true; + bw_stats_topk_ = std::min(32, k); + } + } + } + } + + void destroy() override {} + + int encode(bool, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int encode(int32_t, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int encode(float, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int encode(double, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int encode(common::String, common::ByteStream &) override { + return common::E_TYPE_NOT_MATCH; + } + + int encode(int64_t value, common::ByteStream &out) override { + // Buffer into chunk; encode on flush (whole-stream encoding for benchmarks/tests). + chunk_.push_back(value); + return common::E_OK; + } + + int flush(common::ByteStream &out) override { + if (!chunk_.empty()) { + // Encode remaining chunk; if only 1 value, write single-value block. + if ((int)chunk_.size() == 1) { + write_marker_if_needed(out); + // [0][0][preValue] + common::SerializationUtil::write_ui8(0, out); + common::SerializationUtil::write_ui8(0, out); + common::SerializationUtil::write_ui64((uint64_t)chunk_[0], out); + chunk_.clear(); + } else { + int ret = encode_chunk(out); + if (ret != common::E_OK) return ret; + } + } + + // Trailing legacy path for values_ is unused here; keep for parity. + if (!values_.empty()) { + // Encode any pending as RLE (rare). Format: [0][size] + RLE payload + common::SerializationUtil::write_ui8(0, out); + common::SerializationUtil::write_int_little_endian_padded_on_bit_width( + (int32_t)values_.size(), out, 1); + Int64RleEncoder rle; + for (auto v : values_) { + rle.encode(v, out); + } + rle.flush(out); + values_.clear(); + } + if (collect_bw_stats_ && bw_hist_total_blocks_ > 0) { + print_bw_topk(); + } + reset(); + return common::E_OK; + } + + int get_max_byte_size() override { return (int)(1 + (chunk_.size() + values_.size() + 2) * 8); } + + private: + static FORCE_INLINE uint64_t u64_max_upto_32_scalar(const int64_t *p, int n) { + uint64_t m = 0; + for (int i = 0; i < n; i++) m = std::max(m, (uint64_t)p[i]); + return m; + } + + static FORCE_INLINE uint64_t u64_max_upto_32_neon(const int64_t *p, int n) { + // Best-effort SIMD for arm64; falls back to scalar. + uint64_t m = 0; +#if defined(__ARM_NEON) + int i = 0; + uint64x2_t vmax2 = vdupq_n_u64(0); + for (; i + 1 < n; i += 2) { + // Treat residuals as unsigned magnitudes (they are non-negative after zigzag). + uint64x2_t v = vld1q_u64(reinterpret_cast(p + i)); + // Some toolchains don't expose vmaxq_u64; implement via compare+select. + uint64x2_t mask = vcgtq_u64(v, vmax2); + vmax2 = vbslq_u64(mask, v, vmax2); + } + uint64_t lanes[2]; + vst1q_u64(lanes, vmax2); + m = std::max(lanes[0], lanes[1]); + for (; i < n; i++) m = std::max(m, (uint64_t)p[i]); +#else + for (int i = 0; i < n; i++) m = std::max(m, (uint64_t)p[i]); +#endif + return m; + } + + int encode_chunk(common::ByteStream &out) { + if ((int)chunk_.size() < 2) { + // defer to values_ for flush + for (auto v : chunk_) values_.push_back(v); + chunk_.clear(); + return common::E_OK; + } + write_marker_if_needed(out); + + const int n = (int)chunk_.size(); + // originals: use chunk_ itself. + residuals_.resize((size_t)(n - 1)); + fire_pred_.reset(); + int64_t pre = chunk_[0]; + for (int i = 1; i < n; i++) { + int64_t pred = (predict_method_ == "delta") ? (chunk_[i] - pre) + : fire_predict(chunk_[i], pre); + residuals_[i - 1] = zigzag(pred); + pre = chunk_[i]; + } + + int pack_size = use_optimal_ + ? optimal::SprintzOptimalPackSize::find_optimal_pack_size( + residuals_.data(), + (int)residuals_.size(), + &bw_scratch_, + enable_simd_) + : fixed_pack_size_; + pack_size = std::max(1, std::min(32, pack_size)); + + const int rlen = (int)residuals_.size(); + const int num_packs = (rlen + pack_size - 1) / pack_size; + + packer_n_.set_width(1); + + for (int p = 0; p < num_packs; p++) { + int start = p * pack_size; + int end = std::min(start + pack_size, rlen); + int actual_pack = end - start; + // preValue is originals[start] + int64_t block_pre = chunk_[start]; + + uint64_t maxv = 0; + if (enable_simd_) { + maxv = u64_max_upto_32_neon(residuals_.data() + start, actual_pack); + } else { + maxv = u64_max_upto_32_scalar(residuals_.data() + start, actual_pack); + } + maxv = std::max(1, maxv); + int bit_width = 64 - __builtin_clzll(maxv); + if (bit_width <= 0) bit_width = 1; + if (collect_bw_stats_) { + int bw = std::max(1, std::min(64, bit_width)); + bw_hist_[(size_t)bw]++; + bw_hist_total_blocks_++; + } + + int packed_bytes = (actual_pack * bit_width + 7) / 8; + packer_n_.set_width(bit_width); + // packed bytes upper bound for n<=32 and width<=64: 256 bytes + packer_n_.pack_n_values(residuals_.data(), start, actual_pack, packed_scratch_); + + common::SerializationUtil::write_ui8((uint8_t)actual_pack, out); + common::SerializationUtil::write_ui8((uint8_t)bit_width, out); + common::SerializationUtil::write_ui64((uint64_t)block_pre, out); + out.write_buf(packed_scratch_, (uint32_t)packed_bytes); + } + + chunk_.clear(); + return common::E_OK; + } + + void print_bw_topk() const { + struct Pair { + int bw; + uint32_t cnt; + }; + std::vector v; + v.reserve(64); + for (int bw = 1; bw <= 64; bw++) { + uint32_t c = bw_hist_[(size_t)bw]; + if (c) v.push_back(Pair{bw, c}); + } + std::sort(v.begin(), v.end(), [](const Pair &a, const Pair &b) { + if (a.cnt != b.cnt) return a.cnt > b.cnt; + return a.bw < b.bw; + }); + int k = std::min(bw_stats_topk_, (int)v.size()); + std::cout << "[opt-bw-topk] blocks=" << (unsigned long long)bw_hist_total_blocks_ + << " topk=" << k << "\n"; + for (int i = 0; i < k; i++) { + double pct = 100.0 * (double)v[i].cnt / (double)bw_hist_total_blocks_; + std::cout << " bw=" << v[i].bw << " cnt=" << (unsigned long long)v[i].cnt + << " pct=" << pct << "\n"; + } + std::cout.flush(); + } + + void write_marker_if_needed(common::ByteStream &out) { + if (marker_written_) return; + common::SerializationUtil::write_ui8(0, out); + common::SerializationUtil::write_ui8(1, out); + marker_written_ = true; + } + + int64_t fire_predict(int64_t value, int64_t prev) { + int64_t pred = fire_pred_.predict(prev); + int64_t err = value - pred; + fire_pred_.train(prev, value, err); + return err; + } + + static int64_t zigzag(int64_t pred) { return (pred <= 0) ? (-2 * pred) : (2 * pred - 1); } + + private: + bool use_optimal_; + int fixed_pack_size_; + std::string predict_method_; + int optimal_chunk_min_size_; + bool enable_simd_; + + LongFire fire_pred_; + std::vector values_; + std::vector chunk_; + bool marker_written_; + std::vector bw_scratch_; + std::vector residuals_; + uint8_t packed_scratch_[256]; + bitpacking::Int64PackerN packer_n_; + + // Optional light stats (Optimal only; enabled by env TSFILE_OPT_BW_TOPK) + bool collect_bw_stats_ = false; + int bw_stats_topk_ = 0; + std::array bw_hist_{}; + uint64_t bw_hist_total_blocks_ = 0; +}; + +class SprintzOptInt64Decoder final : public Decoder { + public: + explicit SprintzOptInt64Decoder(const std::string &predict_method = "delta", + bool enable_simd_unpack = true) + : predict_method_(predict_method), + unpacker_(1, enable_simd_unpack), + fire_pred_(3) { + reset(); + } + + void reset() override { + determined_ = false; + optimal_mode_ = false; + current_.clear(); + idx_ = 0; + fire_pred_.reset(); + emit_pre_value_ = true; + } + + bool has_remaining(const common::ByteStream &buffer) override { + return idx_ < current_.size() || buffer.has_remaining(); + } + + int read_boolean(bool &, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int read_int32(int32_t &, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int read_float(float &, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int read_double(double &, common::ByteStream &) override { return common::E_TYPE_NOT_MATCH; } + int read_String(common::String &, common::PageArena &, common::ByteStream &) override { + return common::E_TYPE_NOT_MATCH; + } + + int read_int64(int64_t &ret_value, common::ByteStream &in) override { + if (idx_ >= current_.size()) { + int ret = decode_next_block(in); + if (ret != common::E_OK) return ret; + } + if (idx_ >= current_.size()) return common::E_NO_MORE_DATA; + ret_value = current_[idx_++]; + return common::E_OK; + } + + private: + int decode_next_block(common::ByteStream &in) { + if (!determined_) { + // Peek first two bytes: if [0][1] -> optimal mode, else rewind not supported in ByteStream, + // so we require optimal mode in this codec. + uint8_t b0 = 0, b1 = 0; + int ret = common::SerializationUtil::read_ui8(b0, in); + if (ret != common::E_OK) return ret; + ret = common::SerializationUtil::read_ui8(b1, in); + if (ret != common::E_OK) return ret; + optimal_mode_ = (b0 == 0 && b1 == 1); + determined_ = true; + if (!optimal_mode_) { + return common::E_OUT_OF_RANGE; + } + } + + uint8_t pack_size_u8 = 0; + int ret = common::SerializationUtil::read_ui8(pack_size_u8, in); + if (ret != common::E_OK) return ret; + int pack_size = (int)pack_size_u8; + if (pack_size == 0) { + uint8_t next = 0; + ret = common::SerializationUtil::read_ui8(next, in); + if (ret != common::E_OK) return ret; + if (next == 0) { + uint64_t pre = 0; + ret = common::SerializationUtil::read_ui64(pre, in); + if (ret != common::E_OK) return ret; + current_.assign(1, (int64_t)pre); + idx_ = 0; + emit_pre_value_ = false; + return common::E_OK; + } + // RLE block [0][size] + payload + int size = (int)next; + current_.resize(size); + Int64RleDecoder rle; + for (int i = 0; i < size; i++) { + int64_t v; + ret = rle.read_int64(v, in); + if (ret != common::E_OK) return ret; + current_[i] = v; + } + idx_ = 0; + emit_pre_value_ = false; + return common::E_OK; + } + + pack_size = std::min(pack_size, 32); + uint8_t bit_width_u8 = 0; + ret = common::SerializationUtil::read_ui8(bit_width_u8, in); + if (ret != common::E_OK) return ret; + int32_t bit_width = (int32_t)bit_width_u8; + uint64_t pre_u64 = 0; + ret = common::SerializationUtil::read_ui64(pre_u64, in); + if (ret != common::E_OK) return ret; + int packed_bytes = (pack_size * bit_width + 7) / 8; + if (packed_bytes > (int)sizeof(packed_scratch_) || pack_size > 32) { + return common::E_OUT_OF_RANGE; + } + uint32_t packed_read = 0; + ret = in.read_buf(packed_scratch_, (uint32_t)packed_bytes, packed_read); + if (ret != common::E_OK) return ret; + if (packed_read != (uint32_t)packed_bytes) { + return common::E_PARTIAL_READ; + } + + unpacker_.set_width(bit_width); + unpacker_.unpack_n_values(packed_scratch_, 0, pack_size, pack_residuals_); + + // reconstruct originals: recon[0]=pre, then zigzag inverse + predictor (match encoder). + recon_vals_[0] = (int64_t)pre_u64; + if (predict_method_ == "delta") { + for (int i = 0; i < pack_size; i++) { + uint64_t z = (uint64_t)pack_residuals_[i]; + int64_t e = (z % 2 == 0) ? -(int64_t)(z / 2) : (int64_t)((z + 1) / 2); + recon_vals_[i + 1] = recon_vals_[i] + e; + } + } else { + fire_pred_.reset(); + for (int i = 0; i < pack_size; i++) { + uint64_t z = (uint64_t)pack_residuals_[i]; + int64_t e = (z % 2 == 0) ? -(int64_t)(z / 2) : (int64_t)((z + 1) / 2); + int64_t pred = fire_pred_.predict(recon_vals_[i]); + int64_t v = pred + e; + recon_vals_[i + 1] = v; + fire_pred_.train(recon_vals_[i], v, e); + } + } + + if (emit_pre_value_) { + current_.resize((size_t)pack_size + 1); + std::memcpy(current_.data(), recon_vals_, + sizeof(int64_t) * (size_t)(pack_size + 1)); + emit_pre_value_ = false; + } else { + current_.resize((size_t)pack_size); + std::memcpy(current_.data(), recon_vals_ + 1, + sizeof(int64_t) * (size_t)pack_size); + } + idx_ = 0; + return common::E_OK; + } + + private: + std::string predict_method_; + bitpacking::Int64PackerN unpacker_; + LongFire fire_pred_; + + uint8_t packed_scratch_[256]; + int64_t pack_residuals_[32]; + int64_t recon_vals_[33]; + + bool determined_; + bool optimal_mode_; + std::vector current_; + size_t idx_; + bool emit_pre_value_; +}; + +} // namespace storage + +#endif // ENCODING_SPRINTZ_OPT_INT64_CODEC_H + diff --git a/cpp/test/encoding/sprintz_opt_int64_codec_test.cc b/cpp/test/encoding/sprintz_opt_int64_codec_test.cc new file mode 100644 index 000000000..b9afbd1a5 --- /dev/null +++ b/cpp/test/encoding/sprintz_opt_int64_codec_test.cc @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include + +#include "common/allocator/byte_stream.h" +#include "encoding/sprintz_opt_int64_codec.h" + +using storage::SprintzOptInt64Decoder; +using storage::SprintzOptInt64Encoder; + +namespace { + +static std::vector gen_data(int n) { + std::mt19937_64 rng(42); + std::normal_distribution dist(0.0, 1000.0); + std::vector out; + out.reserve(n); + int64_t base = 1000000; + for (int i = 0; i < n; i++) { + out.push_back(base + (int64_t)dist(rng)); + } + return out; +} + +TEST(SprintzOptInt64CodecTest, RoundTripFixedPackSize8) { + // On Apple M1 (arm64), we expect NEON to be available in release builds. + // This does NOT guarantee speedup; it just confirms the SIMD path is compiled in. + ASSERT_TRUE(SprintzOptInt64Encoder::kNeonEnabled); + auto data = gen_data(4096); + common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); + SprintzOptInt64Encoder enc(/*use_optimal=*/false, /*fixed_pack_size=*/8, "delta", 256); + for (auto v : data) { + ASSERT_EQ(enc.encode(v, stream), common::E_OK); + } + ASSERT_EQ(enc.flush(stream), common::E_OK); + ASSERT_GT(stream.total_size(), 0u); + + SprintzOptInt64Decoder dec("delta"); + std::vector decoded; + decoded.reserve(data.size()); + char *buf = common::get_bytes_from_bytestream(stream); + ASSERT_NE(buf, nullptr); + common::ByteStream in(1024, common::MOD_DECODER_OBJ); + ASSERT_EQ(in.write_buf((const uint8_t *)buf, stream.total_size()), common::E_OK); + ASSERT_GT(in.remaining_size(), 0u); + { + char hdr[2] = {0, 0}; + uint32_t rl = 0; + int ret = in.read_buf(hdr, 2, rl); + ASSERT_EQ(ret, common::E_OK); + ASSERT_EQ(rl, 2u); + // recreate 'in' from scratch for actual decode + in.reset(); + ASSERT_EQ(in.write_buf((const uint8_t *)buf, stream.total_size()), common::E_OK); + } + for (size_t i = 0; i < data.size(); i++) { + int64_t v; + ASSERT_EQ(dec.read_int64(v, in), common::E_OK); + decoded.push_back(v); + } + free(buf); + ASSERT_EQ(decoded, data); +} + +TEST(SprintzOptInt64CodecTest, RoundTripOptimalPackSize) { + auto data = gen_data(4096); + common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); + SprintzOptInt64Encoder enc(/*use_optimal=*/true, /*fixed_pack_size=*/8, "delta", 256); + for (auto v : data) { + ASSERT_EQ(enc.encode(v, stream), common::E_OK); + } + ASSERT_EQ(enc.flush(stream), common::E_OK); + ASSERT_GT(stream.total_size(), 0u); + + SprintzOptInt64Decoder dec("delta"); + std::vector decoded; + decoded.reserve(data.size()); + char *buf = common::get_bytes_from_bytestream(stream); + ASSERT_NE(buf, nullptr); + common::ByteStream in(1024, common::MOD_DECODER_OBJ); + ASSERT_EQ(in.write_buf((const uint8_t *)buf, stream.total_size()), common::E_OK); + ASSERT_GT(in.remaining_size(), 0u); + { + char hdr[2] = {0, 0}; + uint32_t rl = 0; + int ret = in.read_buf(hdr, 2, rl); + ASSERT_EQ(ret, common::E_OK); + ASSERT_EQ(rl, 2u); + in.reset(); + ASSERT_EQ(in.write_buf((const uint8_t *)buf, stream.total_size()), common::E_OK); + } + for (size_t i = 0; i < data.size(); i++) { + int64_t v; + ASSERT_EQ(dec.read_int64(v, in), common::E_OK); + decoded.push_back(v); + } + free(buf); + ASSERT_EQ(decoded, data); +} + +} // namespace + diff --git a/cpp/test/encoding/sprintz_optimal_pack_size_test.cc b/cpp/test/encoding/sprintz_optimal_pack_size_test.cc new file mode 100644 index 000000000..21dee85ca --- /dev/null +++ b/cpp/test/encoding/sprintz_optimal_pack_size_test.cc @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include + +#include "encoding/optimal/sprintz_optimal_pack_size.h" + +using storage::optimal::SprintzOptimalPackSize; + +namespace { + +static int64_t cost_for_pack_size(const std::vector &vals, int p) { + const int bits_overhead = 80; + const int n = (int)vals.size(); + const int m = (n + p - 1) / p; + int64_t cost = (int64_t)m * bits_overhead; + for (int i = 0; i < m; i++) { + int start = i * p; + int end = std::min(start + p, n); + int max_bw = 1; + for (int j = start; j < end; j++) { + uint64_t v = (uint64_t)std::max(1, vals[j]); + int bw = 64 - __builtin_clzll(v); + max_bw = std::max(max_bw, bw); + } + cost += (int64_t)(end - start) * (int64_t)max_bw; + } + return cost; +} + +TEST(SprintzOptimalPackSizeTest, MatchesBruteforceOnSmallInputs) { + std::vector> cases = { + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + {1, 2, 3, 4, 5, 6, 7, 8, 9}, + {1, 255, 1, 255, 1, 255, 1, 255, 1, 255, 1, 255}, + {1, 1, 1, 1, 1024, 1024, 1024, 1024, 1, 1, 1, 1}, + }; + + for (const auto &vals : cases) { + int n = (int)vals.size(); + int best = SprintzOptimalPackSize::find_optimal_pack_size(vals.data(), n, nullptr); + best = std::max(1, std::min(32, best)); + + int max_p = std::min(32, n); + int best_bf = 1; + int64_t best_cost = INT64_MAX; + for (int p = 1; p <= max_p; p++) { + int64_t c = cost_for_pack_size(vals, p); + if (c < best_cost) { + best_cost = c; + best_bf = p; + } + } + + ASSERT_EQ(best, best_bf); + } +} + +} // namespace + diff --git a/cpp/test/encoding/sprintz_packsize8_vs_optimal_benchmark_test.cc b/cpp/test/encoding/sprintz_packsize8_vs_optimal_benchmark_test.cc new file mode 100644 index 000000000..a1199a6e4 --- /dev/null +++ b/cpp/test/encoding/sprintz_packsize8_vs_optimal_benchmark_test.cc @@ -0,0 +1,651 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/allocator/byte_stream.h" +#include "encoding/sprintz_opt_int64_codec.h" + +using storage::SprintzOptInt64Decoder; +using storage::SprintzOptInt64Encoder; + +namespace { + +struct Result { + std::string dataset; + std::string mode; + int64_t points; + int64_t tsfile_size_bytes; + int64_t write_encode_ns; + int64_t write_io_ns; + int64_t read_io_ns; + int64_t read_decode_ns; +}; + +static bool ends_with(const std::string &s, const std::string &suffix) { + return s.size() >= suffix.size() && + s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +static bool dataset_selected(const std::string &name) { + // Optional: comma-separated allowlist, e.g. "City-temp.csv,Food-price.csv" + if (const char *env = std::getenv("TSFILE_DATASET_NAMES")) { + std::string s(env); + if (s.empty()) return true; + size_t pos = 0; + while (pos < s.size()) { + while (pos < s.size() && (s[pos] == ' ' || s[pos] == '\t' || s[pos] == ',')) pos++; + size_t start = pos; + while (pos < s.size() && s[pos] != ',') pos++; + size_t end = pos; + while (end > start && (s[end - 1] == ' ' || s[end - 1] == '\t')) end--; + if (end > start) { + if (name == s.substr(start, end - start)) return true; + } + pos++; // skip comma + } + return false; + } + return true; +} + +static std::vector read_and_scale_csv_as_int64(const std::string &csv_path) { + std::ifstream in(csv_path); + if (!in.is_open()) { + return {}; + } + std::vector nums; + nums.reserve(1 << 16); + int decimal_max = 0; + + std::string line; + while (std::getline(in, line)) { + std::stringstream ss(line); + std::string cell; + while (std::getline(ss, cell, ',')) { + // trim spaces + size_t b = cell.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) continue; + size_t e = cell.find_last_not_of(" \t\r\n"); + std::string s = cell.substr(b, e - b + 1); + if (s.empty()) continue; + nums.push_back(s); + auto dot = s.find('.'); + if (dot != std::string::npos && dot + 1 < s.size()) { + int dec = (int)(s.size() - (dot + 1)); + decimal_max = std::max(decimal_max, dec); + } + } + } + + auto pow10 = [](int k) -> long double { + long double r = 1.0; + for (int i = 0; i < k; i++) r *= 10.0; + return r; + }; + long double scale = pow10(decimal_max); + + std::vector out; + out.reserve(nums.size()); + for (const auto &s : nums) { + char *endp = nullptr; + long double v = strtold(s.c_str(), &endp); + if (endp == s.c_str()) continue; // skip non-numeric + // Match Java path: scale -> int32 overflow semantics -> widen to int64. + // Java uses int[] scaledInts_all, so values wrap in 32-bit signed range. + long double scaled_ld = llround(v * scale); + int64_t scaled_i64 = (int64_t)scaled_ld; // implementation-defined on overflow; keep as-is + int32_t scaled_i32 = (int32_t)scaled_i64; // wrap like Java int cast + out.push_back((int64_t)scaled_i32); + } + return out; +} + +static int getenv_int_default(const char *key, int default_val) { + if (const char *e = std::getenv(key)) { + if (e[0] == '\0') return default_val; + return std::max(0, atoi(e)); + } + return default_val; +} + +// Sprintz compression SIMD (Neon pack / optimal layout): default off. TSFILE_SPRINTZ_ENCODE_SIMD=1 to enable. +static bool sprintz_encode_simd_enabled() { + if (const char *e = std::getenv("TSFILE_SPRINTZ_ENCODE_SIMD")) { + return atoi(e) != 0; + } + return false; +} + +// Bit unpack SIMD: default on (independent of encode). TSFILE_SPRINTZ_DECODE_SIMD=0 to disable. +static bool sprintz_decode_simd_enabled() { + if (const char *e = std::getenv("TSFILE_SPRINTZ_DECODE_SIMD")) { + return atoi(e) != 0; + } + return true; +} + +// Default ON: cold-cache friendly IO + stronger write flush (can push write_io toward ~50% vs encode on fast SSD). +static bool benchmark_cold_io_enabled() { + if (const char *e = std::getenv("TSFILE_BENCHMARK_COLD_IO")) { + return atoi(e) != 0; + } + return true; +} + +static bool benchmark_write_o_sync() { + if (const char *e = std::getenv("TSFILE_WRITE_O_SYNC")) { + return atoi(e) != 0; + } + return benchmark_cold_io_enabled(); +} + +// 0 = one write + final sync only. When cold IO: default 64 KiB per write/read syscall batch. +static size_t write_fsync_chunk_bytes() { + int kb = getenv_int_default("TSFILE_WRITE_FSYNC_CHUNK_KB", benchmark_cold_io_enabled() ? 64 : 0); + return kb > 0 ? (size_t)kb * 1024u : (size_t)0; +} + +// Timed reads use the same per-chunk size as timed writes (TSFILE_WRITE_FSYNC_CHUNK_KB). +static size_t read_chunk_bytes() { + return write_fsync_chunk_bytes(); +} + +static int read_thrash_megabytes() { + if (!benchmark_cold_io_enabled()) return 0; + return getenv_int_default("TSFILE_READ_THRASH_MB", 256); +} + +// Writes spend most time in O_SYNC + per-chunk F_FULLFSYNC; a single plain read is much faster. +// Optionally repeat full-file reads (still timed) until cumulative read_io reaches this % of write_io. +// 0 = one read only. 100 ~= match write duration (bounded by TSFILE_READ_IO_MATCH_MAX_PASSES). +static int read_io_match_write_pct() { + if (!benchmark_cold_io_enabled()) return 0; + return std::min(100, getenv_int_default("TSFILE_READ_IO_MATCH_WRITE_PCT", 100)); +} + +static int read_io_match_max_passes() { + return std::max(1, getenv_int_default("TSFILE_READ_IO_MATCH_MAX_PASSES", 4096)); +} + +static bool sync_fd_heavy(int fd) { +#ifdef F_FULLFSYNC + if (fcntl(fd, F_FULLFSYNC) != 0) return false; +#else + if (fsync(fd) != 0) return false; +#endif + return true; +} + +// One-time large file used to pollute page cache before timed reads (best-effort). +static void ensure_thrash_file(const std::string &dir, int mb) { + if (mb <= 0) return; + static bool created = false; + if (created) return; + std::string path = dir + "/.benchmark_cache_thrash.bin"; + const size_t total = (size_t)mb * 1024u * 1024u; + const size_t chunk = 4u * 1024u * 1024u; + int fd = ::open(path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644); + if (fd < 0) return; +#ifdef F_NOCACHE + (void)fcntl(fd, F_NOCACHE, 1); +#endif + std::vector buf(chunk, 0); + for (size_t off = 0; off < total; off += chunk) { + size_t n = std::min(chunk, total - off); + size_t woff = 0; + while (woff < n) { + ssize_t w = ::write(fd, buf.data(), n - woff); + if (w <= 0) { + ::close(fd); + return; + } + woff += (size_t)w; + } + (void)sync_fd_heavy(fd); + } + (void)sync_fd_heavy(fd); + if (::close(fd) != 0) return; + created = true; +} + +static void read_thrash_file_untimed(const std::string &dir, int mb) { + if (mb <= 0) return; + std::string path = dir + "/.benchmark_cache_thrash.bin"; + int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) return; +#ifdef F_NOCACHE + (void)fcntl(fd, F_NOCACHE, 1); +#endif + struct stat st; + if (fstat(fd, &st) != 0) { + ::close(fd); + return; + } + const size_t sz = (size_t)st.st_size; + size_t io_chunk = read_chunk_bytes(); + if (io_chunk == 0) { + io_chunk = 64u * 1024u; + } + std::vector tmp(io_chunk); + volatile unsigned char sink = 0; + size_t off = 0; + while (off < sz) { + size_t want = std::min(io_chunk, sz - off); + ssize_t r = ::read(fd, tmp.data(), want); + if (r <= 0) break; + for (ssize_t i = 0; i < r; i++) sink ^= (unsigned char)tmp[(size_t)i]; + off += (size_t)r; + } + (void)sink; + ::close(fd); +} + +static std::vector repeat_dataset(const std::vector &in, int repeat) { + if (repeat <= 1 || in.empty()) { + return in; + } + std::vector out; + out.reserve(in.size() * (size_t)repeat); + for (int i = 0; i < repeat; i++) { + out.insert(out.end(), in.begin(), in.end()); + } + return out; +} + +static bool write_whole_file(const std::string &path, const char *buf, size_t nbytes) { + int flags = O_CREAT | O_TRUNC | O_WRONLY; +#ifdef O_SYNC + if (benchmark_write_o_sync()) flags |= O_SYNC; +#endif + int fd = ::open(path.c_str(), flags, 0644); + if (fd < 0) return false; +#ifdef F_NOCACHE + (void)fcntl(fd, F_NOCACHE, 1); +#endif + const size_t chunk = write_fsync_chunk_bytes(); + size_t off = 0; + if (chunk == 0 || !benchmark_cold_io_enabled()) { + while (off < nbytes) { + ssize_t w = ::write(fd, buf + off, nbytes - off); + if (w <= 0) { + ::close(fd); + return false; + } + off += (size_t)w; + } + } else { + while (off < nbytes) { + size_t want = std::min(chunk, nbytes - off); + size_t woff = 0; + while (woff < want) { + ssize_t w = ::write(fd, buf + off + woff, want - woff); + if (w <= 0) { + ::close(fd); + return false; + } + woff += (size_t)w; + } + off += want; + if (benchmark_cold_io_enabled() && !sync_fd_heavy(fd)) { + ::close(fd); + return false; + } + } + } + // Final metadata / durability flush. With per-chunk sync, one more sync is enough; without chunking, + // use an extra F_FULLFSYNC on macOS in cold mode to push write_io time up. + if (!sync_fd_heavy(fd)) { + ::close(fd); + return false; + } +#ifdef F_FULLFSYNC + if (benchmark_cold_io_enabled() && chunk == 0) { + (void)fcntl(fd, F_FULLFSYNC); + } +#endif + ::close(fd); + return true; +} + +static bool dir_exists(const std::string &path) { + struct stat st; + return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); +} + +static void ensure_dir(const std::string &path) { + // best-effort: single-level mkdir; path here is absolute and should already exist mostly. + mkdir(path.c_str(), 0755); +} + +static bool read_whole_file(const std::string &path, std::vector &out) { + int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) return false; +#ifdef F_NOCACHE + (void)fcntl(fd, F_NOCACHE, 1); +#endif + struct stat st; + if (fstat(fd, &st) != 0) { + ::close(fd); + return false; + } + size_t sz = (size_t)st.st_size; + out.resize(sz); + const size_t chunk = read_chunk_bytes(); + size_t off = 0; + if (chunk == 0 || !benchmark_cold_io_enabled()) { + while (off < sz) { + ssize_t r = ::read(fd, out.data() + off, sz - off); + if (r <= 0) { + ::close(fd); + return false; + } + off += (size_t)r; + } + } else { + while (off < sz) { + size_t want = std::min(chunk, sz - off); + size_t roff = 0; + while (roff < want) { + ssize_t r = ::read(fd, out.data() + off + roff, want - roff); + if (r <= 0) { + ::close(fd); + return false; + } + roff += (size_t)r; + } + off += want; + } + } + ::close(fd); + return true; +} + +static Result run_once_one_dataset(const std::string &dataset_name, + const std::vector &data, + bool optimal_mode, + const std::string &tmp_dir, + int64_t iter_id) { + const std::string mode = optimal_mode ? "OptimalPackSize" : "PackSize8"; + + common::ByteStream out(1024, common::MOD_ENCODER_OBJ); + SprintzOptInt64Encoder enc(/*use_optimal=*/optimal_mode, + /*fixed_pack_size=*/8, + "delta", + 512, + /*enable_simd=*/sprintz_encode_simd_enabled()); + + auto t0 = std::chrono::high_resolution_clock::now(); + for (auto v : data) { + enc.encode(v, out); + } + enc.flush(out); + auto t1 = std::chrono::high_resolution_clock::now(); + + int64_t write_encode_ns = + std::chrono::duration_cast(t1 - t0).count(); + + uint32_t encoded_size = out.total_size(); + char *encoded_buf = common::get_bytes_from_bytestream(out); + EXPECT_NE(encoded_buf, nullptr); + + // Single whole-file I/O timing (as requested) + // Per request (B): use unique filenames per repeat to reduce caching artifacts. + // Keep mode in filename so PackSize8 and Optimal don't collide. + std::string out_path = tmp_dir + "/" + dataset_name + + (optimal_mode ? "_optimal_" : "_pack8_") + + std::to_string((long long)iter_id) + ".tsfile.bin"; + + auto w0 = std::chrono::high_resolution_clock::now(); + bool w_ok = write_whole_file(out_path, encoded_buf, encoded_size); + auto w1 = std::chrono::high_resolution_clock::now(); + EXPECT_TRUE(w_ok); + int64_t write_io_ns = + std::chrono::duration_cast(w1 - w0).count(); + + // Best-effort cold read: thrash page cache with a large unrelated file (un-timed). + read_thrash_file_untimed(tmp_dir, read_thrash_megabytes()); + + std::vector file_bytes; + int64_t read_io_ns = 0; + int match_pct = read_io_match_write_pct(); + int64_t read_target_ns = match_pct > 0 ? (write_io_ns * (int64_t)match_pct) / 100 : 0; + int max_read_passes = read_io_match_max_passes(); + int read_pass = 0; + do { + auto rp0 = std::chrono::high_resolution_clock::now(); + bool r_ok = read_whole_file(out_path, file_bytes); + auto rp1 = std::chrono::high_resolution_clock::now(); + EXPECT_TRUE(r_ok); + read_io_ns += + std::chrono::duration_cast(rp1 - rp0).count(); + read_pass++; + } while (match_pct > 0 && read_io_ns < read_target_ns && read_pass < max_read_passes); + + // Decode timing (in-memory) + common::ByteStream in_stream(1024, common::MOD_DECODER_OBJ); + if (!file_bytes.empty()) { + in_stream.write_buf((const uint8_t *)file_bytes.data(), (uint32_t)file_bytes.size()); + } + SprintzOptInt64Decoder dec("delta", sprintz_decode_simd_enabled()); + std::vector decoded; + decoded.reserve(data.size()); + + auto t2 = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < data.size(); i++) { + int64_t v; + int ret = dec.read_int64(v, in_stream); + EXPECT_EQ(ret, common::E_OK); + decoded.push_back(v); + } + auto t3 = std::chrono::high_resolution_clock::now(); + + EXPECT_EQ(decoded, data); + + int64_t read_decode_ns = + std::chrono::duration_cast(t3 - t2).count(); + + // Per request: delete generated file immediately to avoid disk bloat. + (void)unlink(out_path.c_str()); + + free(encoded_buf); + + return Result{dataset_name, + mode, + (int64_t)data.size(), + (int64_t)encoded_size, + write_encode_ns, + write_io_ns, + read_io_ns, + read_decode_ns}; +} + +static Result avg_results(const std::vector &rs) { + Result out = rs[0]; + out.write_encode_ns = 0; + out.write_io_ns = 0; + out.read_io_ns = 0; + out.read_decode_ns = 0; + for (const auto &r : rs) { + out.tsfile_size_bytes = r.tsfile_size_bytes; // last + out.write_encode_ns += r.write_encode_ns; + out.write_io_ns += r.write_io_ns; + out.read_io_ns += r.read_io_ns; + out.read_decode_ns += r.read_decode_ns; + } + int64_t n = (int64_t)rs.size(); + out.write_encode_ns /= n; + out.write_io_ns /= n; + out.read_io_ns /= n; + out.read_decode_ns /= n; + return out; +} + +static void append_csv(const std::string &path, const std::vector &rows) { + bool exists = false; + { + std::ifstream fin(path); + exists = fin.good(); + } + std::ofstream out(path, std::ios::app); + if (!exists) { + out << "Dataset,Mode,TsFile Size (bytes),Write Time (ns),Write Encode (ns),Write IO (ns)," + "Read Time (ns),Read IO (ns),Read Decode (ns),Points,Compression Ratio\n"; + } + for (const auto &r : rows) { + double raw_bytes = (double)r.points * 8.0; + double ratio = (double)r.tsfile_size_bytes / raw_bytes; + int64_t write_time = r.write_encode_ns + r.write_io_ns; + int64_t read_time = r.read_io_ns + r.read_decode_ns; + out << r.dataset << "," << r.mode << "," << r.tsfile_size_bytes << "," << write_time << "," + << r.write_encode_ns << "," << r.write_io_ns << "," << read_time << "," << r.read_io_ns + << "," << r.read_decode_ns << "," << r.points << "," << ratio << "\n"; + } +} + +TEST(SprintzPackSize8VsOptimal, CsvBenchmarkFairOrder) { + // Match Java test data directory (adjust if needed). + const std::string directory = + "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + const std::string output_dir = + "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_tsfile_packsize_comparison_cpp"; + ensure_dir(output_dir); + const std::string csv_path = output_dir + "/tsfile_comparison_cpp.csv"; + + ASSERT_TRUE(dir_exists(directory)); + // Overwrite CSV on every run (avoid appending across runs). + (void)unlink(csv_path.c_str()); + + ensure_thrash_file(output_dir, read_thrash_megabytes()); + + const int warmup_repeats = + std::max(0, getenv_int_default("TSFILE_BENCHMARK_WARMUP", 0)); + const int measure_repeats = + std::max(1, getenv_int_default("TSFILE_BENCHMARK_MEASURE_REPEATS", 1)); + const int dataset_repeat = 100; // expand each dataset by 100x, as requested + int dataset_limit = 0; // 0 = no limit + if (const char *env = std::getenv("TSFILE_DATASET_LIMIT")) { + dataset_limit = std::max(0, atoi(env)); + } + bool stats_only = false; + if (const char *env = std::getenv("TSFILE_OPT_BW_STATS_ONLY")) { + stats_only = (atoi(env) != 0); + } + + DIR *dp = opendir(directory.c_str()); + ASSERT_NE(dp, (DIR *)nullptr); + struct dirent *de; + int datasets_done = 0; + while ((de = readdir(dp)) != nullptr) { + std::string name = de->d_name; + if (name == "." || name == "..") continue; + if (!ends_with(name, ".csv")) continue; + if (!dataset_selected(name)) continue; + + auto data = read_and_scale_csv_as_int64(directory + "/" + name); + if (data.empty()) continue; + data = repeat_dataset(data, dataset_repeat); + + std::cout << "[benchmark] dataset=" << name << " points=" << (long long)data.size() << std::endl; + + if (stats_only) { + // Only collect Optimal bitWidth top-k distribution; skip IO/timing. + common::ByteStream out(1024, common::MOD_ENCODER_OBJ); + SprintzOptInt64Encoder enc(/*use_optimal=*/true, + /*fixed_pack_size=*/8, + "delta", + 512, + /*enable_simd=*/sprintz_encode_simd_enabled()); + for (auto v : data) { + enc.encode(v, out); + } + enc.flush(out); + datasets_done++; + if (dataset_limit > 0 && datasets_done >= dataset_limit) break; + continue; + } + + // warmup: Pack8 -> Opt, then Opt -> Pack8 (balanced) + for (int w = 0; w < warmup_repeats; w++) { + std::cout << "[warmup] PackSize8 dataset=" << name << " iter=" << w << std::endl; + (void)run_once_one_dataset(name, data, /*optimal=*/false, output_dir, w); + std::cout << "[warmup] OptimalPackSize dataset=" << name << " iter=" << w << std::endl; + (void)run_once_one_dataset(name, data, /*optimal=*/true, output_dir, w); + } + for (int w = 0; w < warmup_repeats; w++) { + std::cout << "[warmup] OptimalPackSize dataset=" << name << " iter=" << (w + warmup_repeats) << std::endl; + (void)run_once_one_dataset(name, data, /*optimal=*/true, output_dir, w + warmup_repeats); + std::cout << "[warmup] PackSize8 dataset=" << name << " iter=" << (w + warmup_repeats) << std::endl; + (void)run_once_one_dataset(name, data, /*optimal=*/false, output_dir, w + warmup_repeats); + } + + std::vector pack8_samples; + std::vector opt_samples; + pack8_samples.reserve(2 * measure_repeats); + opt_samples.reserve(2 * measure_repeats); + + for (int m = 0; m < measure_repeats; m++) { + int64_t iter_id = (int64_t)m; + std::cout << "[measure] PackSize8 dataset=" << name << " iter=" << (long long)iter_id << std::endl; + pack8_samples.push_back( + run_once_one_dataset(name, data, /*optimal=*/false, output_dir, iter_id)); + std::cout << "[measure] OptimalPackSize dataset=" << name << " iter=" << (long long)iter_id << std::endl; + opt_samples.push_back( + run_once_one_dataset(name, data, /*optimal=*/true, output_dir, iter_id)); + } + for (int m = 0; m < measure_repeats; m++) { + int64_t iter_id = (int64_t)(m + measure_repeats); + std::cout << "[measure] OptimalPackSize dataset=" << name << " iter=" << (long long)iter_id << std::endl; + opt_samples.push_back( + run_once_one_dataset(name, data, /*optimal=*/true, output_dir, iter_id)); + std::cout << "[measure] PackSize8 dataset=" << name << " iter=" << (long long)iter_id << std::endl; + pack8_samples.push_back( + run_once_one_dataset(name, data, /*optimal=*/false, output_dir, iter_id)); + } + + auto pack8_avg = avg_results(pack8_samples); + auto opt_avg = avg_results(opt_samples); + append_csv(csv_path, {pack8_avg, opt_avg}); + + datasets_done++; + if (dataset_limit > 0 && datasets_done >= dataset_limit) { + break; + } + } + closedir(dp); +} + +} // namespace + diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index ad0c2bd09..6fc5ea07d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -165,6 +165,19 @@ public class TSFileConfig implements Serializable { /** When true, each block automatically finds optimal pack size for bit-packing (new algorithm). */ private boolean sprintzUseOptimalPackSize = false; + /** + * Minimum number of values buffered before optimal Sprintz runs pack-size search and flushes a + * chunk. Larger values amortize search and allocation overhead (faster writes); smaller values + * allow finer-grained per-chunk pack sizes. Default 1024. + */ + private int sprintzOptimalChunkMinSize = 1024; + + /** + * When true, each page in a chunk body is written via a separate {@code TsFileOutput.write} (and + * flush). Intended for experiments that attribute disk I/O per page; default false. + */ + private boolean writeChunkBodyOneStreamWritePerPage = false; + /** Default frequency type is SINGLE_FREQ. */ private String freqType = "SINGLE_FREQ"; @@ -723,6 +736,24 @@ public void setSprintzUseOptimalPackSize(boolean sprintzUseOptimalPackSize) { this.sprintzUseOptimalPackSize = sprintzUseOptimalPackSize; } + public int getSprintzOptimalChunkMinSize() { + return sprintzOptimalChunkMinSize; + } + + public void setSprintzOptimalChunkMinSize(int sprintzOptimalChunkMinSize) { + // Allow down to 8 for experiments; cap to limit memory use of encoder scratch buffers. + this.sprintzOptimalChunkMinSize = + Math.max(8, Math.min(1 << 20, sprintzOptimalChunkMinSize)); + } + + public boolean isWriteChunkBodyOneStreamWritePerPage() { + return writeChunkBodyOneStreamWritePerPage; + } + + public void setWriteChunkBodyOneStreamWritePerPage(boolean writeChunkBodyOneStreamWritePerPage) { + this.writeChunkBodyOneStreamWritePerPage = writeChunkBodyOneStreamWritePerPage; + } + public String getHdfsFile() { return hdfsFile; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java index 45f202efa..935c8a78c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java @@ -23,17 +23,17 @@ import org.apache.tsfile.encoding.fire.LongFire; import org.apache.tsfile.encoding.optimal.SprintzOptimalPackSize; import org.apache.tsfile.exception.encoding.TsFileEncodingException; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Vector; public class LongSprintzEncoder extends SprintzEncoder { - private static final int OPTIMAL_CHUNK_MIN_SIZE = 32; private static final byte OPTIMAL_MODE_MARKER = 0; private static final byte OPTIMAL_MODE_VERSION = 1; @@ -41,8 +41,31 @@ public class LongSprintzEncoder extends SprintzEncoder { LongFire firePred; protected Vector values; - /** For optimal mode: buffer to collect values before finding optimal pack size. */ - private final ArrayList chunkBuffer = new ArrayList<>(); + /** For optimal mode: primitive buffer (avoids per-point Long boxing). */ + private long[] chunkLongBuffer = new long[1024]; + + private int chunkLen = 0; + + /** Reused buffers to avoid per-chunk long[] allocation in optimal mode. */ + private long[] optimalScratchOriginals = new long[0]; + + private long[] optimalScratchResiduals = new long[0]; + + /** Reused bit-width scratch for {@link SprintzOptimalPackSize} (same length as residual count). */ + private int[] optimalBitWidthScratch = new int[0]; + + /** Actual pack size is at most 32; reuse for bit-packing scratch. */ + private final long[] optimalPackScratch = new long[32]; + + /** Reuse packed output (max 32 * 64 bits). */ + private byte[] optimalPackedScratch = new byte[256]; + + private final byte[] preValueBytesScratch = new byte[8]; + + /** Avoid constructing a new {@link LongPacker} on every sub-pack when bit width repeats. */ + private LongPacker reusedOptimalPacker; + + private int reusedOptimalPackerWidth = -1; /** For optimal mode: whether we've written the mode marker. */ private boolean optimalModeMarkerWritten = false; @@ -58,10 +81,35 @@ public LongSprintzEncoder() { protected void reset() { super.reset(); values.clear(); - chunkBuffer.clear(); + chunkLen = 0; optimalModeMarkerWritten = false; } + private void chunkAdd(long value) { + if (chunkLen >= chunkLongBuffer.length) { + chunkLongBuffer = + Arrays.copyOf(chunkLongBuffer, Math.max(chunkLen + 1, chunkLongBuffer.length * 2)); + } + chunkLongBuffer[chunkLen++] = value; + } + + private void ensureOptimalScratch(int nValues) { + if (optimalScratchOriginals.length < nValues) { + optimalScratchOriginals = new long[nValues]; + optimalScratchResiduals = new long[Math.max(0, nValues - 1)]; + } + } + + private void ensureOptimalBitWidthScratch(int nResiduals) { + if (optimalBitWidthScratch.length < nResiduals) { + optimalBitWidthScratch = new int[nResiduals]; + } + } + + private int optimalChunkMinSize() { + return config.getSprintzOptimalChunkMinSize(); + } + @Override public int getOneItemMaxSize() { return 1 + (1 + 32) * Long.BYTES; @@ -69,7 +117,7 @@ public int getOneItemMaxSize() { @Override public long getMaxByteSize() { - return 1 + (1L + values.size() + chunkBuffer.size()) * Long.BYTES; + return 1 + (1L + values.size() + chunkLen) * Long.BYTES; } protected Long predict(Long value, Long preVlaue) throws TsFileEncodingException { @@ -128,7 +176,9 @@ private void encodeChunkWithOptimalPackSize(long[] originals, long[] residuals) return; } - int packSize = SprintzOptimalPackSize.findOptimalPackSize(residuals); + ensureOptimalBitWidthScratch(n); + int packSize = + SprintzOptimalPackSize.findOptimalPackSize(residuals, n, optimalBitWidthScratch); packSize = Math.max(1, Math.min(32, packSize)); int numPacks = (n + packSize - 1) / packSize; @@ -139,30 +189,45 @@ private void encodeChunkWithOptimalPackSize(long[] originals, long[] residuals) int actualPackSize = end - start; long preValue = originals[start]; - long[] packResiduals = new long[actualPackSize]; for (int i = 0; i < actualPackSize; i++) { - packResiduals[i] = residuals[start + i]; + optimalPackScratch[i] = residuals[start + i]; } - int packBitWidth = getLongArrayMaxBitWidth(packResiduals); + int packBitWidth = getLongArrayMaxBitWidth(optimalPackScratch, actualPackSize); packBitWidth = Math.max(1, packBitWidth); - packer = new LongPacker(packBitWidth); int packedBytes = (actualPackSize * packBitWidth + 7) / 8; - byte[] bytes = new byte[packedBytes]; - packer.packNValues(packResiduals, 0, actualPackSize, bytes); + if (optimalPackedScratch.length < packedBytes) { + optimalPackedScratch = new byte[packedBytes]; + } + LongPacker pkr = optimalPackerFor(packBitWidth); + pkr.packNValues(optimalPackScratch, 0, actualPackSize, optimalPackedScratch); byteCache.write(actualPackSize); ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth(packBitWidth, byteCache, 1); - byteCache.write(ByteBuffer.allocate(8).putLong(preValue).array()); - byteCache.write(bytes, 0, bytes.length); + BytesUtils.longToBytes(preValue, preValueBytesScratch, 0); + byteCache.write(preValueBytesScratch, 0, 8); + byteCache.write(optimalPackedScratch, 0, packedBytes); + } + } + + private LongPacker optimalPackerFor(int bitWidth) { + if (reusedOptimalPackerWidth != bitWidth) { + reusedOptimalPacker = new LongPacker(bitWidth); + reusedOptimalPackerWidth = bitWidth; } + return reusedOptimalPacker; } private static int getLongArrayMaxBitWidth(long[] arr) { + return getLongArrayMaxBitWidth(arr, arr.length); + } + + private static int getLongArrayMaxBitWidth(long[] arr, int len) { int max = 1; - for (long num : arr) { - int bw = 64 - Long.numberOfLeadingZeros(Math.max(1, num)); + for (int i = 0; i < len; i++) { + long num = arr[i]; + int bw = 64 - Long.numberOfLeadingZeros(Math.max(1L, num)); max = Math.max(max, bw); } return max; @@ -170,7 +235,7 @@ private static int getLongArrayMaxBitWidth(long[] arr) { @Override public void flush(ByteArrayOutputStream out) throws IOException { - if (config.isSprintzUseOptimalPackSize() && !chunkBuffer.isEmpty()) { + if (config.isSprintzUseOptimalPackSize() && chunkLen > 0) { encodeRemainingChunkWithOptimal(); } if (byteCache.size() > 0) { @@ -228,19 +293,20 @@ public void flush(ByteArrayOutputStream out) throws IOException { } private void encodeRemainingChunkWithOptimal() throws IOException { - if (chunkBuffer.size() < 2) { - values.addAll(chunkBuffer); - chunkBuffer.clear(); + if (chunkLen < 2) { + for (int i = 0; i < chunkLen; i++) { + values.add(chunkLongBuffer[i]); + } + chunkLen = 0; return; } - int n = chunkBuffer.size(); - long[] originals = new long[n]; - for (int i = 0; i < n; i++) { - originals[i] = chunkBuffer.get(i); - } + int n = chunkLen; + ensureOptimalScratch(n); + long[] originals = optimalScratchOriginals; + System.arraycopy(chunkLongBuffer, 0, originals, 0, n); - long[] residuals = new long[n - 1]; + long[] residuals = optimalScratchResiduals; firePred.reset(); long pre = originals[0]; for (int i = 1; i < n; i++) { @@ -258,7 +324,7 @@ private void encodeRemainingChunkWithOptimal() throws IOException { optimalModeMarkerWritten = true; } encodeChunkWithOptimalPackSize(originals, residuals); - chunkBuffer.clear(); + chunkLen = 0; } @Override @@ -271,9 +337,9 @@ public void encode(long value, ByteArrayOutputStream out) { } private void encodeOptimalMode(long value, ByteArrayOutputStream out) { - chunkBuffer.add(value); + chunkAdd(value); - if (chunkBuffer.size() >= OPTIMAL_CHUNK_MIN_SIZE) { + if (chunkLen >= optimalChunkMinSize()) { try { if (!optimalModeMarkerWritten) { byteCache.write(OPTIMAL_MODE_MARKER); @@ -281,13 +347,12 @@ private void encodeOptimalMode(long value, ByteArrayOutputStream out) { optimalModeMarkerWritten = true; } - int n = chunkBuffer.size(); - long[] originals = new long[n]; - for (int i = 0; i < n; i++) { - originals[i] = chunkBuffer.get(i); - } + int n = chunkLen; + ensureOptimalScratch(n); + long[] originals = optimalScratchOriginals; + System.arraycopy(chunkLongBuffer, 0, originals, 0, n); - long[] residuals = new long[n - 1]; + long[] residuals = optimalScratchResiduals; firePred.reset(); long pre = originals[0]; for (int i = 1; i < n; i++) { @@ -301,7 +366,7 @@ private void encodeOptimalMode(long value, ByteArrayOutputStream out) { } encodeChunkWithOptimalPackSize(originals, residuals); - chunkBuffer.clear(); + chunkLen = 0; groupNum++; if (groupNum == groupMax) { // Write accumulated chunks to out but do NOT call full flush(): reset() would set diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java index 6c606439c..b00fb6507 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/optimal/SprintzOptimalPackSize.java @@ -20,24 +20,35 @@ package org.apache.tsfile.encoding.optimal; /** - * Utility for finding optimal pack size for Sprintz bit-packing encoding using RMQ (Range Maximum - * Query) sparse table. Minimizes total storage cost: sum(pack_size * max_bitwidth_in_pack) + - * num_packs * BITS_PER_BLOCK_OVERHEAD. Each block in the encoder writes packSize(1 byte) + - * bitWidth(1 byte) + preValue(8 bytes) = 80 bits overhead, so we use 80 in the cost model. + * Utility for finding optimal pack size for Sprintz bit-packing encoding. Minimizes total storage + * cost: sum(pack_size * max_bitwidth_in_pack) + num_packs * BITS_PER_BLOCK_OVERHEAD. Each block in + * the encoder writes packSize(1 byte) + bitWidth(1 byte) + preValue(8 bytes) = 80 bits overhead, + * so we use 80 in the cost model. + * + *

Implementation: for each candidate pack size p in [1, min(32, n)], scan segments in O(n) with + * no auxiliary heap allocation (avoids sparse-table alloc/GC on every chunk). Overall O(32 * n), + * which is fast for typical chunk sizes and much cheaper than building an RMQ table per call. */ public class SprintzOptimalPackSize { private SprintzOptimalPackSize() {} /** - * Find optimal pack size for long values (e.g., Sprintz residuals) using RMQ-based cost - * minimization. + * Find optimal pack size for long values (e.g., Sprintz residuals) using cost minimization. * * @param values array of non-negative long values (e.g., after Sprintz predict transform) * @return optimal pack size in range [1, n] */ public static int findOptimalPackSize(long[] values) { - int n = values.length; + return findOptimalPackSize(values, values.length, null); + } + + /** + * Same as {@link #findOptimalPackSize(long[])} but uses only {@code values[0..n)} and optionally + * reuses {@code bwScratch} (length >= n) to store per-element bit widths, avoiding extra + * allocation and redundant {@code numberOfLeadingZeros} work across pack-size candidates. + */ + public static int findOptimalPackSize(long[] values, int n, int[] bwScratch) { if (n <= 0) { return 1; } @@ -45,67 +56,34 @@ public static int findOptimalPackSize(long[] values) { return Math.max(1, n); } - // Build sparse table for RMQ on bit widths - int[] bitWidths = new int[n]; - long globalMax = 0; + int[] bw = bwScratch != null && bwScratch.length >= n ? bwScratch : new int[n]; for (int i = 0; i < n; i++) { long value = values[i]; - if (value > globalMax) { - globalMax = value; - } - bitWidths[i] = 64 - Long.numberOfLeadingZeros(Math.max(1, value)); + bw[i] = 64 - Long.numberOfLeadingZeros(Math.max(1L, value)); } // Per-block overhead in encoder: 1 byte packSize + 1 byte bitWidth + 8 bytes preValue = 80 bits final int bitsPerBlockOverhead = 80; - int logN = 32 - Integer.numberOfLeadingZeros(n); - int[][] st = new int[logN][n]; - - for (int i = 0; i < n; i++) { - st[0][i] = bitWidths[i]; - } - - for (int k = 1; k < logN; k++) { - int step = 1 << (k - 1); - for (int i = 0; i + (1 << k) <= n; i++) { - st[k][i] = Math.max(st[k - 1][i], st[k - 1][i + step]); - } - } - - int[] log2 = new int[n + 1]; - for (int i = 2; i <= n; i++) { - log2[i] = log2[i / 2] + 1; - } - int bestPackSize = 1; long bestCost = Long.MAX_VALUE; - int maxPackSize = Math.min(32, n); // encoder caps pack size at 32 + int maxPackSize = Math.min(32, n); for (int p = 1; p <= maxPackSize; p++) { int m = (n + p - 1) / p; long cost = 0; - for (int i = 0; i < m - 1; i++) { + for (int i = 0; i < m; i++) { int start = i * p; - int end = start + p - 1; - int k = log2[p]; - int maxBitWidth = - Math.max(st[k][start], st[k][end - (1 << k) + 1]); - cost += (long) p * maxBitWidth; - } - - if (m > 0) { - int lastStart = (m - 1) * p; - int lastEnd = n - 1; - int r = n - lastStart; - - if (r > 0) { - int k = log2[r]; - int lastMaxBitWidth = - Math.max(st[k][lastStart], st[k][lastEnd - (1 << k) + 1]); - cost += (long) r * lastMaxBitWidth; + int end = Math.min(start + p, n); + int maxBw = 1; + for (int j = start; j < end; j++) { + int b = bw[j]; + if (b > maxBw) { + maxBw = b; + } } + cost += (long) (end - start) * maxBw; } cost += (long) m * bitsPerBlockOverhead; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ChunkBodyPagedIoWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ChunkBodyPagedIoWriter.java new file mode 100644 index 000000000..45515c797 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ChunkBodyPagedIoWriter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.write.writer; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.PublicBAOS; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * Writes one chunk's serialized page body ({@link PublicBAOS}) using one {@link TsFileOutput#write} + * per page (plus {@link TsFileOutput#flush}). Matches how {@link + * org.apache.tsfile.read.reader.chunk.ChunkReader} walks page headers. + */ +public final class ChunkBodyPagedIoWriter { + + private ChunkBodyPagedIoWriter() {} + + public static void writeChunkBody( + TsFileOutput out, + PublicBAOS bytes, + int numOfPages, + TSDataType dataType, + Statistics chunkStatistics) + throws IOException { + byte[] raw = bytes.toByteArray(); + boolean onlyOnePageChunk = numOfPages <= 1; + ByteBuffer buf = ByteBuffer.wrap(raw); + while (buf.hasRemaining()) { + int sliceStart = buf.position(); + PageHeader ph; + if (onlyOnePageChunk && chunkStatistics != null) { + ph = PageHeader.deserializeFrom(buf, chunkStatistics); + } else { + ph = PageHeader.deserializeFrom(buf, dataType); + } + if (ph.getUncompressedSize() != 0) { + int skip = ph.getCompressedSize(); + int p = buf.position(); + buf.position(p + skip); + } + int sliceEnd = buf.position(); + out.write(Arrays.copyOfRange(raw, sliceStart, sliceEnd)); + out.flush(); + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 96cc383e6..a5c382661 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -136,6 +136,10 @@ public class TsFileIOWriter implements AutoCloseable { private final List flushListeners = new ArrayList<>(); + /** When {@link TSFileConfig#isWriteChunkBodyOneStreamWritePerPage()} is true, used for the next + * {@link #writeBytesToStream(PublicBAOS)} call. */ + private int pendingChunkBodyPagedWriteNumPages = -1; + /** empty construct function. */ protected TsFileIOWriter() { setEncryptParam( @@ -251,7 +255,20 @@ public void addFlushListener(FlushChunkMetadataListener listener) { * @throws IOException if an I/O error occurs. */ public void writeBytesToStream(PublicBAOS bytes) throws IOException { - bytes.writeTo(out.wrapAsStream()); + if (TS_FILE_CONFIG.isWriteChunkBodyOneStreamWritePerPage() + && pendingChunkBodyPagedWriteNumPages >= 0 + && currentChunkMetadata != null) { + ChunkBodyPagedIoWriter.writeChunkBody( + out, + bytes, + pendingChunkBodyPagedWriteNumPages, + currentChunkMetadata.getDataType(), + currentChunkMetadata.getStatistics()); + pendingChunkBodyPagedWriteNumPages = -1; + } else { + pendingChunkBodyPagedWriteNumPages = -1; + bytes.writeTo(out.wrapAsStream()); + } } protected void startFile() throws IOException { @@ -340,6 +357,8 @@ public void startFlushChunk( numOfPages, mask); header.serializeTo(out.wrapAsStream()); + // writeBytesToStream follows only when chunk has body bytes; avoid stale pending for empty chunks. + pendingChunkBodyPagedWriteNumPages = dataSize > 0 ? numOfPages : -1; } /** Write a whole chunk in another file into this file. Providing fast merge for IoTDB. */ diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java index de7acfa9f..5cd70c3b0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -8,6 +8,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; @@ -15,6 +16,8 @@ import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.LongDataPoint; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.schema.Schema; +import org.apache.tsfile.utils.Pair; import org.junit.Assume; import org.junit.Test; @@ -29,6 +32,7 @@ import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Files; import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; @@ -38,13 +42,102 @@ public class AllNo8PacksizeOptimal { static final List IGNORE_FILES = Arrays.asList(".DS_Store", "full_data", "test.csv", "POI-lat.csv", - "POI-lon.csv", "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv"); //,"Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv" + "POI-lon.csv", "Basel-wind.csv", "Basel-temp.csv", "Air-sensor.csv","Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv","init.csv"); //,"Mem-usage.csv","Cpu-usage_right.csv","Disk-usage.csv" static final List NO_TIME_SERIES_FILES = Arrays.asList("Food-price.csv", "electric_vehicle_charging.csv", "POI-lat.csv", "POI-lon.csv", "Blockchain-tr.csv", "SSD-bench.csv", "City-lat.csv","City-lon.csv"); private static final int CHUNK_SIZE = 1024; + /** + * Default repeats for encode/decode nanoTime micro-benchmarks (inner loop between + * {@code startEncodeTime}/{@code encodeDuration} and {@code startDecodeTime}/{@code decodeDuration}, + * see {@link #benchChunkedBitPacking} and {@link #BPTest()}). + */ + private static final int DEFAULT_BENCH_TIME_REPEAT = 100; + + /** One chunk's packed payload for repeated decode timing. */ + private static final class EncodedChunk { + final byte[] compressed; + final int[] bitWidths; + final int packSize; + final int nInts; + + EncodedChunk(byte[] compressed, int[] bitWidths, int packSize, int nInts) { + this.compressed = compressed; + this.bitWidths = bitWidths; + this.packSize = packSize; + this.nInts = nInts; + } + } + + /** Bit-width scan + {@link #encodeBitPackingV2} for one chunk (used inside timed regions). */ + private static EncodedChunk encodeChunkBitPacking(int[] scaledInts, int pack_size) { + pack_size = Math.max(1, pack_size); + int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; + int[] bitWidths = new int[num_of_pack_size]; + for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { + int maxInGroup = 0; + int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); + for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { + if (scaledInts[scaledInts_j] > maxInGroup) { + maxInGroup = scaledInts[scaledInts_j]; + } + } + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[scaledInts_i / pack_size] = bitWidth; + } + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); + return new EncodedChunk(compressedData, bitWidths, pack_size, scaledInts.length); + } + + /** + * For each chunk: {@code time_of_repeat} encode runs (fresh {@code int[]} copy per run so sort/sprintz + * stay correct) averaging {@code startEncodeTime}/{@code encodeDuration}, then {@code time_of_repeat} + * decode runs averaging {@code startDecodeTime}/{@code decodeDuration}. + */ + private static void benchChunkedBitPacking( + int[] scaledInts_all, + int nPoints, + int chunkSize, + int time_of_repeat, + java.util.function.Function encodeChunk, + java.util.function.Consumer decodeChunk, + long[] outCostBits, + long[] outEncodeNs, + long[] outDecodeNs) { + long modelCost = 0; + long modelTime = 0; + long modelDecodeTime = 0; + for (int i = 0; i < nPoints; i += chunkSize) { + int end = Math.min(i + chunkSize, nPoints); + int len = end - i; + + EncodedChunk last = null; + long startEncodeTime = System.nanoTime(); + for (int rep = 0; rep < time_of_repeat; rep++) { + int[] chunk = new int[len]; + System.arraycopy(scaledInts_all, i, chunk, 0, len); + last = encodeChunk.apply(chunk); + if (rep == 0) { + modelCost += last.compressed.length * 8L; + } + } + long encodeDuration = System.nanoTime() - startEncodeTime; + modelTime += (encodeDuration/ time_of_repeat); + + long startDecodeTime = System.nanoTime(); + for (int rep = 0; rep < time_of_repeat; rep++) { + decodeChunk.accept(last); + } + long decodeDuration = System.nanoTime() - startDecodeTime; + modelDecodeTime += (decodeDuration/ time_of_repeat); + } + outCostBits[0] = modelCost; + outEncodeNs[0] = modelTime; + outDecodeNs[0] = modelDecodeTime; + } + public static int getCount(long long1, int mask) { return ((int) (long1 & mask)); } @@ -2642,8 +2735,7 @@ public static int[] sprintzDecode(int[] encodedData) { - @Test - public void OptimalPackSizeBPAllNo8Test() throws IOException { + public static void main(String[] args) throws IOException { System.out.println("\nPerformance Testing..."); String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_BP_all_no8"; @@ -2689,7 +2781,7 @@ public void OptimalPackSizeBPAllNo8Test() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -2713,59 +2805,22 @@ public void OptimalPackSizeBPAllNo8Test() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV2(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, findOptimalPackSizeallV2(chunk)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -4080,7 +4135,7 @@ public void BPTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -4104,59 +4159,22 @@ public void BPTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = 16; - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, 8), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -4260,7 +4278,7 @@ public void Simple8bTest() throws IOException { throw new AssertionError("Simple8b round-trip mismatch: " + file.getName()); } - int timeOfRepeat = 50; + int timeOfRepeat = 100; long modelCostBits = 0; long modelTimeNs = 0; long modelDecodeTimeNs = 0; @@ -4534,8 +4552,10 @@ private static int[] simple8bRoundTripInts(int[] values) { return longsToIntsZigZag(decoded); } - // ---------------- Simple8b (Anh & Moffat 2010) ---------------- - // selector: 0..15 in top 4 MSBs, payload: remaining 60 bits + // ---------------- Simple8b (Anh & Moffat 2010, Table I) ---------------- + // 4-bit selector in bits 0..3 (LSB); 60 data bits in bits 4..63. Selectors 0 and 1 code + // runs of 240 resp. 120 consecutive values equal to 1 (1-origin gaps in the paper). + private static final long SIMPLE8B_SELECTOR_MASK = 0xFL; private static final int[] SIMPLE8B_BITS = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; private static final int[] SIMPLE8B_N = @@ -4590,7 +4610,7 @@ private static long[] simple8bDecodeAll(long[] words, int valueCount) { long[] out = new long[valueCount]; int outPos = 0; for (long word : words) { - int selector = (int) (word >>> 60); + int selector = (int) (word & SIMPLE8B_SELECTOR_MASK); int n = SIMPLE8B_N[selector]; if (outPos + n > valueCount) { throw new IllegalArgumentException("Decoded past expected valueCount"); @@ -4603,27 +4623,34 @@ private static long[] simple8bDecodeAll(long[] words, int valueCount) { return out; } - private static int simple8bSelect(long[] src, int off, int remaining) { - int scan = Math.min(remaining, 240); - long max = 0; - boolean allZero = true; - for (int i = 0; i < scan; i++) { - long v = src[off + i]; - if (v != 0) { - allZero = false; - if (v > max) max = v; + private static boolean simple8bGroupFits(long[] src, int off, int selector, int remaining) { + int n = SIMPLE8B_N[selector]; + int bits = SIMPLE8B_BITS[selector]; + if (n > remaining) { + return false; + } + if (bits == 0) { + for (int i = 0; i < n; i++) { + if (src[off + i] != 1L) { + return false; + } + } + return true; + } + long mask = SIMPLE8B_MASK[selector]; + for (int i = 0; i < n; i++) { + if ((src[off + i] & ~mask) != 0) { + return false; } } - int neededBits = (max == 0) ? 0 : (64 - Long.numberOfLeadingZeros(max)); + return true; + } + + private static int simple8bSelect(long[] src, int off, int remaining) { for (int selector = 0; selector < 16; selector++) { - int n = SIMPLE8B_N[selector]; - if (n > remaining) continue; - int bits = SIMPLE8B_BITS[selector]; - if (bits == 0) { - if (allZero) return selector; - continue; + if (simple8bGroupFits(src, off, selector, remaining)) { + return selector; } - if (neededBits <= bits) return selector; } return 15; } @@ -4631,28 +4658,30 @@ private static int simple8bSelect(long[] src, int off, int remaining) { private static long simple8bPack(long[] src, int off, int selector) { int bits = SIMPLE8B_BITS[selector]; int n = SIMPLE8B_N[selector]; - long word = ((long) selector) << 60; - if (bits == 0) return word; + long word = selector & SIMPLE8B_SELECTOR_MASK; + if (bits == 0) { + return word; + } long mask = SIMPLE8B_MASK[selector]; for (int i = 0; i < n; i++) { - int shift = 60 - bits * (i + 1); + int shift = 4 + bits * i; word |= ((src[off + i] & mask) << shift); } return word; } private static int simple8bUnpack(long word, long[] dst, int off) { - int selector = (int) (word >>> 60); + int selector = (int) (word & SIMPLE8B_SELECTOR_MASK); int bits = SIMPLE8B_BITS[selector]; int n = SIMPLE8B_N[selector]; if (bits == 0) { - Arrays.fill(dst, off, off + n, 0L); + Arrays.fill(dst, off, off + n, 1L); return n; } long mask = SIMPLE8B_MASK[selector]; for (int i = 0; i < n; i++) { - int shift = 60 - bits * (i + 1); + int shift = 4 + bits * i; dst[off + i] = (word >>> shift) & mask; } return n; @@ -4707,7 +4736,7 @@ public void OptimalPackSizeN2Test() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -4731,59 +4760,22 @@ public void OptimalPackSizeN2Test() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV2(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, findOptimalPackSizeallV2(chunk)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -4819,6 +4811,7 @@ public void OptimalPackSizeN2SortTest() throws IOException { for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + if (!NO_TIME_SERIES_FILES.contains(file.getName())) continue; System.out.println(file.getName()); String Output = outputDirstr + "/" + file.getName(); CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); @@ -4854,7 +4847,7 @@ public void OptimalPackSizeN2SortTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT;//; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -4878,60 +4871,25 @@ public void OptimalPackSizeN2SortTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - quickSortDesc(scaledInts,0,scaledInts.length - 1); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallForSort(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + quickSortDesc(chunk, 0, chunk.length - 1); + return encodeChunkBitPacking(chunk, findOptimalPackSizeallForSort(chunk)); + }, + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5004,7 +4962,7 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5028,61 +4986,26 @@ public void OptimalPackSizeN2SprintzTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV2(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + int pack_size = Math.max(1, findOptimalPackSizeallV2(scaledInts)); + return encodeChunkBitPacking(scaledInts, pack_size); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5153,7 +5076,7 @@ public void OptimalPackSizeN2SprintzSortTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5177,62 +5100,26 @@ public void OptimalPackSizeN2SprintzSortTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - quickSortDesc(scaledInts,0,scaledInts.length - 1); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallForSort(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + quickSortDesc(scaledInts, 0, scaledInts.length - 1); + return encodeChunkBitPacking(scaledInts, findOptimalPackSizeallForSort(scaledInts)); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5303,7 +5190,7 @@ public void OptimalPackSizeRMQTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5327,59 +5214,22 @@ public void OptimalPackSizeRMQTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, findOptimalPackSizeallV3(chunk)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5451,7 +5301,7 @@ public void OptimalPackSizeRMQSortTest() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5475,62 +5325,25 @@ public void OptimalPackSizeRMQSortTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - quickSortDesc(scaledInts,0,scaledInts.length - 1); - // 快速排序排 scaledInts - // Arrays.sort(scaledInts); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3ForSort(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + quickSortDesc(chunk, 0, chunk.length - 1); + return encodeChunkBitPacking(chunk, findOptimalPackSizeallV3ForSort(chunk)); + }, + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5602,7 +5415,7 @@ public void OptimalPackSizeRMQSprintzTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5626,61 +5439,26 @@ public void OptimalPackSizeRMQSprintzTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + int pack_size = Math.max(1, findOptimalPackSizeallV3(scaledInts)); + return encodeChunkBitPacking(scaledInts, pack_size); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -5704,9 +5482,73 @@ public void OptimalPackSizeRMQSprintzTest() throws IOException { } } + /** + * One timed TsFile cycle: in-memory Sprintz encode, then per-segment disk write and read (chunk + * headers / gaps + one segment per page inside chunk bodies), then full decode on the reassembled + * buffer. Pack width 8; {@code optimalPackSize} selects fixed vs optimal pack-size search. + * When {@code acc} is non-null, adds ns to encode, write I/O, read I/O, and decode respectively. + * The test enables {@code TSFileConfig#setWriteChunkBodyOneStreamWritePerPage(boolean)} so the + * in-memory writer also emits one stream write per page. + */ + private static void runTsFileSprintzPackCompareCycle( + boolean optimalPackSize, + File tsfileOut, + IDeviceID deviceID, + List pathList, + long[] dataAsLong, + long[] acc) throws IOException, WriteProcessException { + + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(optimalPackSize); + + if (tsfileOut.exists()) { + //noinspection ResultOfMethodCallIgnored + tsfileOut.delete(); + } + + long te0 = System.nanoTime(); + MemoryTsFileOutput memOut = new MemoryTsFileOutput(); + try (TsFileWriter w = new TsFileWriter(memOut, new Schema())) { + w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); + for (int i = 0; i < dataAsLong.length; i++) { + TSRecord rec = new TSRecord(deviceID, i); + rec.addTuple(new LongDataPoint("sensor_1", dataAsLong[i])); + w.writeRecord(rec); + } + } + long te1 = System.nanoTime(); + byte[] encodedBytes = memOut.toByteArray(); + // Disk write/read: one timed I/O op per segment (chunk headers / gaps + one segment per page). + List ioSegments = TsFilePerPageDiskIoHelper.buildContiguousSegments(encodedBytes); + long writeIoNs = TsFilePerPageDiskIoHelper.writeAllSegmentsTimed(tsfileOut, encodedBytes, ioSegments); + Pair readPair = TsFilePerPageDiskIoHelper.readAllSegmentsTimed(tsfileOut, ioSegments); + byte[] readBuf = readPair.left; + long readIoNs = readPair.right; + long td0 = System.nanoTime(); + try (TsFileReader reader = new TsFileReader(new TsFileSequenceReader(new ByteArrayTsFileInput(readBuf)))) { + QueryDataSet ds = reader.query(QueryExpression.create(pathList, null)); + while (ds.hasNext()) { + ds.next(); + } + } + long td1 = System.nanoTime(); + + if (acc != null) { + acc[0] += (te1 - te0); + acc[1] += writeIoNs; + acc[2] += readIoNs; + acc[3] += (td1 - td0); + } + } + /** * Compare TsFile space, write time, read time: pack size 8 vs optimal pack size. * Writes each dataset to TsFile with Sprintz encoding and measures metrics. + * + *

Fair timing: the same helper drives both modes; warmup and measured rounds use interleaved + * order (Pack8→Optimal then Optimal→Pack8, equal repeats) so neither mode always benefits from + * running immediately after the other (JVM JIT / CPU cache). Averages use {@code 2 * measureRepeats} + * samples per mode. */ @Test public void TsFilePackSize8VsOptimalComparisonTest() throws IOException, WriteProcessException { @@ -5721,14 +5563,34 @@ public void TsFilePackSize8VsOptimalComparisonTest() throws IOException, WritePr String csvPath = outputDirStr + "/tsfile_comparison.csv"; CsvWriter writer = new CsvWriter(csvPath, ',', StandardCharsets.UTF_8); String[] head = { - "Dataset", "Mode", "TsFile Size (bytes)", "Write Time (ns)", "Read Time (ns)", - "Points", "Compression Ratio" + "Dataset", + "Mode", + "TsFile Size (bytes)", + "Baseline Size (bytes)", // ~ timestamp + metadata (+ tiny value) + "Value-only Size (bytes)", // TsFile Size - Baseline Size + "Write Time (ns)", // encode + write-I/O (see split columns) + "Write Encode (ns)", // TsFile build in memory (no disk) + "Write IO (ns)", // per-segment disk write (one op per page + headers/gaps) + "Read Time (ns)", // read file to memory + decode (see split) + "Read IO (ns)", // per-segment disk read (matches write segmentation) + "Read Decode (ns)", // TsFileReader query from in-memory bytes + "Points", + "Compression Ratio", + "Value-only Ratio" }; writer.writeRecord(head); int warmupRepeats = 2; int measureRepeats = 5; + // Reduce metadata overhead: fewer pages/chunks (bigger pages), and keep time encoding efficient for + // monotonic timestamps (i). + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(256 * 1024); + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(200_000); + TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(512 * 1024 * 1024); + TSFileDescriptor.getInstance().getConfig().setWriteChunkBodyOneStreamWritePerPage(true); + + try { for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -5768,94 +5630,128 @@ public void TsFilePackSize8VsOptimalComparisonTest() throws IOException, WritePr Path path = new Path(deviceID, "sensor_1", true); List pathList = Collections.singletonList(path); - // 1. Pack size 8 - TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); - TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + // Pack8 vs Optimal: same code path (runTsFileSprintzPackCompareCycle); interleaved warmup + // and balanced measurement order so neither mode always runs after the other. File tsfile8 = new File(outputDirStr + "/" + file.getName().replace(".csv", "_pack8.tsfile")); + File tsfileOpt = new File(outputDirStr + "/" + file.getName().replace(".csv", "_optimal.tsfile")); + File tsfile8Baseline = + new File(outputDirStr + "/" + file.getName().replace(".csv", "_pack8_baseline.tsfile")); + File tsfileOptBaseline = + new File(outputDirStr + "/" + file.getName().replace(".csv", "_optimal_baseline.tsfile")); if (tsfile8.exists()) tsfile8.delete(); + if (tsfileOpt.exists()) tsfileOpt.delete(); + if (tsfile8Baseline.exists()) tsfile8Baseline.delete(); + if (tsfileOptBaseline.exists()) tsfileOptBaseline.delete(); - long writeTime8 = 0; - long readTime8 = 0; - for (int r = 0; r < warmupRepeats + measureRepeats; r++) { - if (tsfile8.exists()) tsfile8.delete(); - long t0 = System.nanoTime(); - try (TsFileWriter w = new TsFileWriter(tsfile8)) { - w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); - for (int i = 0; i < dataAsLong.length; i++) { - TSRecord rec = new TSRecord(deviceID, i); - rec.addTuple(new LongDataPoint("sensor_1", dataAsLong[i])); - w.writeRecord(rec); - } - } - long t1 = System.nanoTime(); - if (r >= warmupRepeats) writeTime8 += (t1 - t0); + long[] acc8 = new long[4]; + long[] accOpt = new long[4]; + final int timingDenominator = 2 * measureRepeats; - long t2 = System.nanoTime(); - try (TsFileReader reader = new TsFileReader(tsfile8)) { - QueryDataSet ds = reader.query(QueryExpression.create(pathList, null)); - while (ds.hasNext()) ds.next(); - } - long t3 = System.nanoTime(); - if (r >= warmupRepeats) readTime8 += (t3 - t2); + for (int w = 0; w < warmupRepeats; w++) { + runTsFileSprintzPackCompareCycle(false, tsfile8, deviceID, pathList, dataAsLong, null); + runTsFileSprintzPackCompareCycle(true, tsfileOpt, deviceID, pathList, dataAsLong, null); + } + for (int w = 0; w < warmupRepeats; w++) { + runTsFileSprintzPackCompareCycle(true, tsfileOpt, deviceID, pathList, dataAsLong, null); + runTsFileSprintzPackCompareCycle(false, tsfile8, deviceID, pathList, dataAsLong, null); + } + + for (int m = 0; m < measureRepeats; m++) { + runTsFileSprintzPackCompareCycle(false, tsfile8, deviceID, pathList, dataAsLong, acc8); + runTsFileSprintzPackCompareCycle(true, tsfileOpt, deviceID, pathList, dataAsLong, accOpt); } + for (int m = 0; m < measureRepeats; m++) { + runTsFileSprintzPackCompareCycle(true, tsfileOpt, deviceID, pathList, dataAsLong, accOpt); + runTsFileSprintzPackCompareCycle(false, tsfile8, deviceID, pathList, dataAsLong, acc8); + } + long size8 = tsfile8.length(); - long avgWrite8 = writeTime8 / measureRepeats; - long avgRead8 = readTime8 / measureRepeats; + long sizeOpt = tsfileOpt.length(); + + // Baseline (same timestamps, near-zero value entropy) to estimate timestamp+metadata. + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); + TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); + try (TsFileWriter w = new TsFileWriter(tsfile8Baseline)) { + w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); + for (int i = 0; i < dataAsLong.length; i++) { + TSRecord rec = new TSRecord(deviceID, i); + rec.addTuple(new LongDataPoint("sensor_1", 0L)); + w.writeRecord(rec); + } + } + long baseline8 = tsfile8Baseline.length(); + long valueOnly8 = Math.max(0L, size8 - baseline8); + long avgWriteEnc8 = acc8[0] / timingDenominator; + long avgWriteIo8 = acc8[1] / timingDenominator; + long avgReadIo8 = acc8[2] / timingDenominator; + long avgReadDec8 = acc8[3] / timingDenominator; + long avgWrite8 = avgWriteEnc8 + avgWriteIo8; + long avgRead8 = avgReadIo8 + avgReadDec8; double ratio8 = (double) size8 / (numbers.size() * 8.0); + double valueOnlyRatio8 = (double) valueOnly8 / (numbers.size() * 8.0); writer.writeRecord(new String[]{ file.getName(), "PackSize8", - String.valueOf(size8), String.valueOf(avgWrite8), - String.valueOf(avgRead8), String.valueOf(numbers.size()), - String.format("%.4f", ratio8) + String.valueOf(size8), + String.valueOf(baseline8), + String.valueOf(valueOnly8), + String.valueOf(avgWrite8), + String.valueOf(avgWriteEnc8), + String.valueOf(avgWriteIo8), + String.valueOf(avgRead8), + String.valueOf(avgReadIo8), + String.valueOf(avgReadDec8), + String.valueOf(numbers.size()), + String.format("%.4f", ratio8), + String.format("%.4f", valueOnlyRatio8) }); - // 2. Optimal pack size + TSFileDescriptor.getInstance().getConfig().setSprintzBlockSize(8); TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(true); - File tsfileOpt = new File(outputDirStr + "/" + file.getName().replace(".csv", "_optimal.tsfile")); - if (tsfileOpt.exists()) tsfileOpt.delete(); - - long writeTimeOpt = 0; - long readTimeOpt = 0; - for (int r = 0; r < warmupRepeats + measureRepeats; r++) { - if (tsfileOpt.exists()) tsfileOpt.delete(); - long t0 = System.nanoTime(); - try (TsFileWriter w = new TsFileWriter(tsfileOpt)) { - w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); - for (int i = 0; i < dataAsLong.length; i++) { - TSRecord rec = new TSRecord(deviceID, i); - rec.addTuple(new LongDataPoint("sensor_1", dataAsLong[i])); - w.writeRecord(rec); - } - } - long t1 = System.nanoTime(); - if (r >= warmupRepeats) writeTimeOpt += (t1 - t0); - - long t2 = System.nanoTime(); - try (TsFileReader reader = new TsFileReader(tsfileOpt)) { - QueryDataSet ds = reader.query(QueryExpression.create(pathList, null)); - while (ds.hasNext()) ds.next(); - } - long t3 = System.nanoTime(); - if (r >= warmupRepeats) readTimeOpt += (t3 - t2); - } - long sizeOpt = tsfileOpt.length(); - long avgWriteOpt = writeTimeOpt / measureRepeats; - long avgReadOpt = readTimeOpt / measureRepeats; + try (TsFileWriter w = new TsFileWriter(tsfileOptBaseline)) { + w.registerTimeseries(deviceID, new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.SPRINTZ)); + for (int i = 0; i < dataAsLong.length; i++) { + TSRecord rec = new TSRecord(deviceID, i); + rec.addTuple(new LongDataPoint("sensor_1", 0L)); + w.writeRecord(rec); + } + } + long baselineOpt = tsfileOptBaseline.length(); + long valueOnlyOpt = Math.max(0L, sizeOpt - baselineOpt); + long avgWriteEncOpt = accOpt[0] / timingDenominator; + long avgWriteIoOpt = accOpt[1] / timingDenominator; + long avgReadIoOpt = accOpt[2] / timingDenominator; + long avgReadDecOpt = accOpt[3] / timingDenominator; + long avgWriteOpt = avgWriteEncOpt + avgWriteIoOpt; + long avgReadOpt = avgReadIoOpt + avgReadDecOpt; double ratioOpt = (double) sizeOpt / (numbers.size() * 8.0); + double valueOnlyRatioOpt = (double) valueOnlyOpt / (numbers.size() * 8.0); writer.writeRecord(new String[]{ file.getName(), "OptimalPackSize", - String.valueOf(sizeOpt), String.valueOf(avgWriteOpt), - String.valueOf(avgReadOpt), String.valueOf(numbers.size()), - String.format("%.4f", ratioOpt) + String.valueOf(sizeOpt), + String.valueOf(baselineOpt), + String.valueOf(valueOnlyOpt), + String.valueOf(avgWriteOpt), + String.valueOf(avgWriteEncOpt), + String.valueOf(avgWriteIoOpt), + String.valueOf(avgReadOpt), + String.valueOf(avgReadIoOpt), + String.valueOf(avgReadDecOpt), + String.valueOf(numbers.size()), + String.format("%.4f", ratioOpt), + String.format("%.4f", valueOnlyRatioOpt) }); TSFileDescriptor.getInstance().getConfig().setSprintzUseOptimalPackSize(false); - System.out.printf(" %s: Pack8 size=%d bytes, write=%d ns, read=%d ns | Optimal size=%d bytes, write=%d ns, read=%d ns%n", - file.getName(), size8, avgWrite8, avgRead8, - sizeOpt, avgWriteOpt, avgReadOpt); + System.out.printf( + " %s: Pack8 size=%d enc=%d ioW=%d | readIo=%d dec=%d || Opt size=%d enc=%d ioW=%d | readIo=%d dec=%d%n", + file.getName(), size8, avgWriteEnc8, avgWriteIo8, avgReadIo8, avgReadDec8, + sizeOpt, avgWriteEncOpt, avgWriteIoOpt, avgReadIoOpt, avgReadDecOpt); + } + } finally { + TSFileDescriptor.getInstance().getConfig().setWriteChunkBodyOneStreamWritePerPage(false); } writer.close(); System.out.println("Results saved to: " + csvPath); @@ -5909,7 +5805,7 @@ public void OptimalPackSizeRMQSprintzSortTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -5933,64 +5829,26 @@ public void OptimalPackSizeRMQSprintzSortTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - // // 快速排序排 scaledInt - // Arrays.sort(scaledInt); - int[] scaledInts = sprintz(scaledInt); - quickSortDesc(scaledInts,0,scaledInts.length - 1); - // 使用优化的方法找到最优pack_size（现在可以是任意整数） - int pack_size = findOptimalPackSizeallV3ForSort(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunkArg -> { + int[] scaledInts = sprintz(chunkArg); + quickSortDesc(scaledInts, 0, scaledInts.length - 1); + return encodeChunkBitPacking(scaledInts, findOptimalPackSizeallV3ForSort(scaledInts)); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -6067,7 +5925,7 @@ public void VaryPackSizeTest() throws IOException { csvReader.close(); int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; // 缩放数据 int batchSize = 1024; @@ -6088,96 +5946,40 @@ public void VaryPackSizeTest() throws IOException { currentIndex += batch.length; } - // 测试每个pack size + // 测试每个 pack size（与 OptimalPackSizeRMQSprintzSortTest：benchChunkedBitPacking） for (int packSize : packSizes) { System.out.println("Testing pack size: " + packSize); - long totalEncodeTime = 0; - long totalDecodeTime = 0; - long totalCompressedSize = 0; - int totalPoints = 0; - - for (int j = 0; j < time_of_repeat; j++) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - // 编码 - long startEncodeTime = System.nanoTime(); - - // 计算需要的组数 - int numGroups = (scaledInts.length + packSize - 1) / packSize; - int[] bitWidths = new int[numGroups]; - - // 计算每个组的位宽 - for (int group = 0; group < numGroups; group++) { - int startIdx = group * packSize; - int endIdx = Math.min(startIdx + packSize, scaledInts.length); - - int maxInGroup = 0; - for (int idx = startIdx; idx < endIdx; idx++) { - if (scaledInts[idx] > maxInGroup) { - maxInGroup = scaledInts[idx]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[group] = bitWidth; - } - - // 编码数据 - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); - long encodeDuration = System.nanoTime() - startEncodeTime; - - // 解码 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + final int ps = Integer.max(1, packSize); + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, ps), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; + + double compressionRatio = (double) modelCost / (double) (numbers.size() * 64); + double encodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double decodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); -// // 验证解码结果 -// boolean valid = true; -// for (int k = 0; k < scaledInts.length; k++) { -// if (scaledInts[k] != decodedData[k]) { -// System.err.println("Decode error at position " + k + -// ": expected " + scaledInts[k] + ", got " + decodedData[k]); -// valid = false; -// break; -// } -// } -// -// if (!valid) { -// System.err.println("Decoding failed for pack size " + packSize); -// } - - // 累加统计 - totalEncodeTime += encodeDuration; - totalDecodeTime += decodeDuration; - totalCompressedSize += compressedData.length * 8L; // 转换为bits - totalPoints += scaledInts.length; - } - } - - // 计算平均 - long avgEncodeTime = totalEncodeTime / time_of_repeat; - long avgDecodeTime = totalDecodeTime / time_of_repeat; - long avgCompressedSize = totalCompressedSize / time_of_repeat; - totalPoints /= time_of_repeat; - - // 计算吞吐率和压缩率 - double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s - double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s - double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); - - // 写入结果 String[] record = { file.toString(), "BitPacking", String.valueOf(encodeThroughput), String.valueOf(decodeThroughput), - String.valueOf(totalPoints), + String.valueOf(numbers.size()), String.valueOf(packSize), - String.valueOf(avgCompressedSize), + String.valueOf(modelCost), String.valueOf(compressionRatio) }; writer.writeRecord(record); @@ -6819,7 +6621,7 @@ public void OptimalPackSizePruneTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -6843,59 +6645,22 @@ public void OptimalPackSizePruneTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - - int pack_size = findOptimalPackSizeallV6(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, findOptimalPackSizeallV6(chunk)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -6967,7 +6732,7 @@ public void OptimalPackSizePruneSprintzTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -6991,61 +6756,26 @@ public void OptimalPackSizePruneSprintzTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - - int pack_size = findOptimalPackSizeallV6(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + int pack_size = Math.max(1, findOptimalPackSizeallV6(scaledInts)); + return encodeChunkBitPacking(scaledInts, pack_size); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -7137,9 +6867,7 @@ public static void loadRFModel(File scalerFile, File modelFile) throws IOExcepti } } - /** - * Bit-packing encode cost in bits for one chunk (same as OptimalPackSizePruneRMQTest). - */ + public static long encodeChunkBits(int[] scaledInts, int packSize) { int n = scaledInts.length; int numPacks = (n + packSize - 1) / packSize; @@ -7260,11 +6988,7 @@ public static double predictImprovementRF(double[] features) { return sum / rfTreeFeature.size(); } - /** - * Same as OptimalPackSizePruneRMQTest but uses RF-predicted improvement: if predicted gain > 0 use - * findOptimalPackSizeallV5, else pack size 8. Output to output_random_tree. - * Requires export_rf_for_java.py (trained on CompressionImprovementPct). - */ + @Test public void OptimalPackSizeRFPredictTest() throws IOException { String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; @@ -7379,7 +7103,6 @@ public void OptimalPackSizePruneRMQTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); for (File file : Objects.requireNonNull(dir.listFiles())) { -// if(!file.getName().equals("PM10-dust.csv")) continue; if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; System.out.println(file.getName()); @@ -7417,7 +7140,7 @@ public void OptimalPackSizePruneRMQTest() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -7441,59 +7164,22 @@ public void OptimalPackSizePruneRMQTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - - int pack_size = findOptimalPackSizeallV5(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> encodeChunkBitPacking(chunk, findOptimalPackSizeallV5(chunk)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -7514,7 +7200,6 @@ public void OptimalPackSizePruneRMQTest() throws IOException { System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); System.out.println("Compression ratio: " + model_ratio); -// break; } } @@ -7565,7 +7250,7 @@ public void OptimalPackSizePruneRMQSprintzTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -7589,61 +7274,26 @@ public void OptimalPackSizePruneRMQSprintzTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - - int pack_size = findOptimalPackSizeallV5(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] decodedInts = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + int pack_size = Math.max(1, findOptimalPackSizeallV5(scaledInts)); + return encodeChunkBitPacking(scaledInts, pack_size); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -7844,7 +7494,7 @@ public void OptimalPackSizePrunePlusTest() throws IOException { } } } - int time_of_repeat = 500; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -7868,59 +7518,22 @@ public void OptimalPackSizePrunePlusTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInts = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); - - long startTime = System.nanoTime(); - - int pack_size = findOptimalPackSizeallV6Plus(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunkArg -> encodeChunkBitPacking(chunkArg, findOptimalPackSizeallV6Plus(chunkArg)), + ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); @@ -7993,7 +7606,7 @@ public void OptimalPackSizePrunePlusSprintzTest() throws IOException { } } } - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); @@ -8017,61 +7630,26 @@ public void OptimalPackSizePrunePlusSprintzTest() throws IOException { System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); currentIndex += batch.length; } - long modelCost = 0; - long modelTime = 0; - long modelDecodeTime = 0; - - for (int j = 0; j < time_of_repeat; j++) { - int totalCost = 0; - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - long startTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - - int pack_size = findOptimalPackSizeallV6Plus(scaledInts); - - // 确保pack_size至少为1 - pack_size = Math.max(1, pack_size); - - int num_of_pack_size = (scaledInts.length + pack_size - 1) / pack_size; - int[] bitWidths = new int[num_of_pack_size]; - - // 计算每个pack的位宽 - for (int scaledInts_i = 0; scaledInts_i < scaledInts.length; scaledInts_i += pack_size) { - int maxInGroup = 0; - int end_index = Math.min(scaledInts_i + pack_size, scaledInts.length); - for (int scaledInts_j = scaledInts_i; scaledInts_j < end_index; scaledInts_j++) { - if (scaledInts[scaledInts_j] > maxInGroup) { - maxInGroup = scaledInts[scaledInts_j]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[scaledInts_i / pack_size] = bitWidth; - } - - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, pack_size); - long cur_cost = compressedData.length * 8L; // 转换为bit数 - long duration = System.nanoTime() - startTime; - modelTime += (duration); - modelCost += cur_cost; - - // 测试解压性能 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, pack_size, scaledInts.length); - int[] sprintzdata = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - modelDecodeTime += decodeDuration; - - } - - } - modelCost = modelCost / time_of_repeat; - modelTime = (modelTime) / time_of_repeat; - modelDecodeTime = (modelDecodeTime) / time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + int pack_size = Math.max(1, findOptimalPackSizeallV6Plus(scaledInts)); + return encodeChunkBitPacking(scaledInts, pack_size); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; double model_ratio = (double) modelCost / (double) (numbers.size() * 64); double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/ByteArrayTsFileInput.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/ByteArrayTsFileInput.java new file mode 100644 index 000000000..a7e48327d --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/ByteArrayTsFileInput.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding; + +import org.apache.tsfile.read.reader.TsFileInput; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** {@link TsFileInput} over a byte array (no disk I/O) for read-path benchmarking. */ +public final class ByteArrayTsFileInput implements TsFileInput { + + private final byte[] data; + private long position; + + public ByteArrayTsFileInput(byte[] data) { + this.data = data; + this.position = 0L; + } + + @Override + public long size() { + return data.length; + } + + @Override + public long position() { + return position; + } + + @Override + public TsFileInput position(long newPosition) { + this.position = newPosition; + return this; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + if (position >= data.length) { + return -1; + } + int max = (int) Math.min(dst.remaining(), data.length - position); + if (max <= 0) { + return 0; + } + dst.put(data, (int) position, max); + position += max; + return max; + } + + @Override + public int read(ByteBuffer dst, long pos) throws IOException { + if (pos >= data.length) { + return -1; + } + int max = (int) Math.min(dst.remaining(), data.length - pos); + if (max <= 0) { + return 0; + } + dst.put(data, (int) pos, max); + return max; + } + + /** + * Stream reads advance {@link #position} so {@link org.apache.tsfile.read.TsFileSequenceReader} + * sequential parsing (readChunkHeader / readPageHeader / readChunkGroupHeader) stays aligned with + * {@link #read(ByteBuffer)}-based reads. + */ + @Override + public InputStream wrapAsInputStream() { + return new InputStream() { + @Override + public int read() throws IOException { + if (position >= data.length) { + return -1; + } + return data[(int) position++] & 0xff; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (position >= data.length) { + return -1; + } + int n = (int) Math.min(len, data.length - position); + System.arraycopy(data, (int) position, b, off, n); + position += n; + return n; + } + }; + } + + @Override + public void close() { + // no-op + } + + @Override + public String getFilePath() { + return "byte[]://tsfile-benchmark"; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java index c9c57bc72..e5e43152b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/CuSZpCpuTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.*; import java.nio.charset.StandardCharsets; @@ -15,7 +16,22 @@ /** * Simplified CPU reimplementation of a cuSZp-like plain 1D compressor for testing. * - * Workflow: + *

IDE / Cursor debug (CPU only, no GPU): run these first — they finish in seconds: + *

    + *
  • {@link #debugCpuPack8RoundTripSmallSynthetic} + *
  • {@link #debugCpuOptimalV5RoundTripSmallSynthetic} + *
  • {@link #debugCuSZpCpuOneCsvPack8AndV5} — needs camel CSV dir; optional {@code -Dcuszp.debug.csv=City-temp.csv}, + * {@code -Dcuszp.debug.repeats=1} + *
+ * Paths align with {@code fig_alp_cuszp_pack8_vs_v5_combined.py} ({@code BASE} = encoding-pack-size root): + * {@code BASE/ElfTestData_camel}, {@code BASE/output_cuszp_cpu}, {@code BASE/output_cuszp_cpu_optimal_v5}. + * Set repo root: {@code -Dencoding.pack.size.repo=...} or {@code ENCODING_PACK_SIZE_REPO}. + * Optional overrides: {@code -Dcuszp.bench.data.dir=...}, {@code -Dcuszp.bench.output.base=...}, + * {@code -Dcuszp.bench.eb=...} / {@code CUSZP_BENCH_EB} (absolute quant error; default {@code 1e-3}). + * + *

Heavy sweeps (feed the combined figure): {@link #cuSZpCpu1DTest}, {@link #cuSZpCpu1DOptimalV5Test}. + * + *

Workflow: * - read a 1D CSV of numbers (double) * - for each CHUNK, compute delta predictor (prev value) * - quantize with absolute error bound eb: q = round((v - pred) / eb) @@ -31,11 +47,261 @@ public class CuSZpCpuTest { private static final Set IGNORE_FILES = Collections.emptySet(); private static final int CHUNK_SIZE = 8192; // per-iteration chunk size; tune as needed + /** + * Default {@code BASE} = parent of {@code fig_alp_cuszp_pack8_vs_v5_combined.py} + * ({@code OUTPUT_CUSZP_DIR = BASE/output_cuszp_cpu}, etc.). + */ + private static final String DEFAULT_ENCODING_PACK_SIZE_REPO = + "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size"; + + /** Same role as {@code BASE = os.path.dirname(os.path.abspath(__file__))} in the combined figure script. */ + private static File resolveEncodingPackSizeRepo() { + String p = System.getProperty("encoding.pack.size.repo"); + if (p != null && !p.isEmpty()) { + return new File(p); + } + String env = System.getenv("ENCODING_PACK_SIZE_REPO"); + if (env != null && !env.isEmpty()) { + return new File(env); + } + return new File(DEFAULT_ENCODING_PACK_SIZE_REPO); + } + + /** + * {@code BASE/ElfTestData_camel}, unless fully overridden. + */ + private static File resolveDataDir() { + String p = System.getProperty("cuszp.bench.data.dir"); + if (p != null && !p.isEmpty()) { + return new File(p); + } + String env = System.getenv("CUSZP_BENCH_DATA_DIR"); + if (env != null && !env.isEmpty()) { + return new File(env); + } + return new File(resolveEncodingPackSizeRepo(), "ElfTestData_camel"); + } + + /** + * {@code BASE} for output folders. Override only if outputs must leave the repo: + * {@code -Dcuszp.bench.output.base=...} / {@code CUSZP_BENCH_OUTPUT_BASE}. + */ + private static File resolveOutputBaseDir() { + String p = System.getProperty("cuszp.bench.output.base"); + if (p != null && !p.isEmpty()) { + return new File(p); + } + String env = System.getenv("CUSZP_BENCH_OUTPUT_BASE"); + if (env != null && !env.isEmpty()) { + return new File(env); + } + return resolveEncodingPackSizeRepo(); + } + + /** + * Absolute error bound {@code eb} in {@code q = round((v - pred) / eb)} for bench and debug tests. + * Default {@code 1e-3} (looser than legacy {@code 1e-4}, better compression). Override with + * {@code -Dcuszp.bench.eb=...} or {@code CUSZP_BENCH_EB}. + */ + private static double resolveBenchEb() { + String p = System.getProperty("cuszp.bench.eb"); + if (p != null && !p.isEmpty()) { + return Double.parseDouble(p); + } + String env = System.getenv("CUSZP_BENCH_EB"); + if (env != null && !env.isEmpty()) { + return Double.parseDouble(env); + } + return 1e-3; + } + + // ---------- Fast CPU-only tests for Cursor “Debug Test” (no GPU, finish in seconds) ---------- + + /** + * Smoke test: pack=8 encode/decode on synthetic data (no strict error assert; lossy+delta may exceed eb slightly). + */ + @Test + public void debugCpuPack8RoundTripSmallSynthetic() throws Exception { + double eb = resolveBenchEb(); + int packSize = 8; + int n = 3000; + double[] chunk = new double[n]; + for (int i = 0; i < n; i++) { + chunk[i] = Math.sin(i * 0.01) * 100.0; + } + byte[] cmp = encodePlain1D(chunk, eb, packSize); + double[] dec = decodePlain1D(cmp, chunk.length, eb, packSize); + double maxAbs = 0.0; + for (int i = 0; i < chunk.length; i++) { + maxAbs = Math.max(maxAbs, Math.abs(chunk[i] - dec[i])); + } + System.out.println( + "debugCpuPack8: n=" + n + " compressed_bytes=" + cmp.length + " maxAbsErr=" + maxAbs + " eb=" + eb); + } + + /** Smoke test: optimal-pack V5 path on synthetic data (no strict error assert). */ + @Test + public void debugCpuOptimalV5RoundTripSmallSynthetic() throws Exception { + double eb = resolveBenchEb(); + int n = 3000; + double[] chunk = new double[n]; + for (int i = 0; i < n; i++) { + chunk[i] = Math.cos(i * 0.015) * 50.0; + } + byte[] cmp = encodePlain1DOptimalPackV5(chunk, eb); + double[] dec = decodePlain1D(cmp, chunk.length, eb, -1); + double maxAbs = 0.0; + for (int i = 0; i < chunk.length; i++) { + maxAbs = Math.max(maxAbs, Math.abs(chunk[i] - dec[i])); + } + System.out.println( + "debugCpuOptimalV5: n=" + n + " compressed_bytes=" + cmp.length + " maxAbsErr=" + maxAbs + " eb=" + eb); + } + + /** + * One CSV from camel dir, single repeat — fast bench for IDE debug. + * Set {@code -Dcuszp.debug.csv=City-temp.csv} (default {@code test.csv} if present). + */ + @Test + public void debugCuSZpCpuOneCsvPack8AndV5() throws Exception { + File directory = resolveDataDir(); + Assume.assumeTrue("Set cuszp.bench.data.dir or CUSZP_BENCH_DATA_DIR", directory.isDirectory()); + + String only = System.getProperty("cuszp.debug.csv", "test.csv"); + File file = new File(directory, only); + Assume.assumeTrue("Missing " + only + " under " + directory + " (set cuszp.debug.csv)", file.isFile()); + + String outBase = new File(resolveOutputBaseDir(), "output_cuszp_cpu_debug").getAbsolutePath(); + File outputDir = new File(outBase); + if (!outputDir.exists()) { + assertTrue(outputDir.mkdirs() || outputDir.isDirectory()); + } + + double eb = resolveBenchEb(); + int packSize = 8; + int repeats = Integer.parseInt(System.getProperty("cuszp.debug.repeats", "1")); + + List numbers = readCsvToDoubleList(file); + Assume.assumeFalse(numbers.isEmpty()); + + long encNs = 0, decNs = 0, bits = 0; + for (int r = 0; r < repeats; r++) { + int index = 0; + while (index < numbers.size()) { + int end = Math.min(index + CHUNK_SIZE, numbers.size()); + double[] chunk = new double[end - index]; + for (int i = 0; i < chunk.length; i++) { + chunk[i] = numbers.get(index + i); + } + long s1 = System.nanoTime(); + byte[] cmp = encodePlain1D(chunk, eb, packSize); + long e1 = System.nanoTime(); + long s2 = System.nanoTime(); + double[] dec = decodePlain1D(cmp, chunk.length, eb, packSize); + long e2 = System.nanoTime(); + encNs += (e1 - s1); + decNs += (e2 - s2); + bits += (long) cmp.length * 8L; + index = end; + } + } + double points = numbers.size(); + double avgEnc = encNs / (double) repeats; + double avgDec = decNs / (double) repeats; + double avgBits = bits / (double) repeats; + double encMb = (points * 8.0) / avgEnc * 1e3; + double decMb = (points * 8.0) / avgDec * 1e3; + double ratio = avgBits / (points * 64.0); + + String outCsv = outBase + "/" + file.getName().replace(".csv", "_debug_pack8.csv"); + CsvWriter w1 = new CsvWriter(outCsv, ',', StandardCharsets.UTF_8); + try { + w1.writeRecord(new String[] { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size (bits)", + "Compression Ratio" + }); + w1.writeRecord(new String[] { + file.getAbsolutePath(), + "cuSZp-cpu-plain-simplified", + String.valueOf(encMb), + String.valueOf(decMb), + String.valueOf((long) points), + String.valueOf((long) avgBits), + String.valueOf(ratio) + }); + } finally { + w1.close(); + } + System.out.println("debug one CSV pack8: " + file.getName() + " -> " + outCsv + " encMB/s=" + encMb); + + long encNs2 = 0, decNs2 = 0, bits2 = 0; + for (int r = 0; r < repeats; r++) { + int index = 0; + while (index < numbers.size()) { + int end = Math.min(index + CHUNK_SIZE, numbers.size()); + double[] chunk = new double[end - index]; + for (int i = 0; i < chunk.length; i++) { + chunk[i] = numbers.get(index + i); + } + long s1 = System.nanoTime(); + byte[] cmp = encodePlain1DOptimalPackV5(chunk, eb); + long e1 = System.nanoTime(); + long s2 = System.nanoTime(); + double[] dec = decodePlain1D(cmp, chunk.length, eb, -1); + long e2 = System.nanoTime(); + encNs2 += (e1 - s1); + decNs2 += (e2 - s2); + bits2 += (long) cmp.length * 8L; + index = end; + } + } + double avgEnc2 = encNs2 / (double) repeats; + double avgDec2 = decNs2 / (double) repeats; + double avgBits2 = bits2 / (double) repeats; + double encMb2 = (points * 8.0) / avgEnc2 * 1e3; + double decMb2 = (points * 8.0) / avgDec2 * 1e3; + double ratio2 = avgBits2 / (points * 64.0); + String outCsv2 = outBase + "/" + file.getName().replace(".csv", "_debug_v5.csv"); + CsvWriter w2 = new CsvWriter(outCsv2, ',', StandardCharsets.UTF_8); + try { + w2.writeRecord(new String[] { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size (bits)", + "Compression Ratio" + }); + w2.writeRecord(new String[] { + file.getAbsolutePath(), + "cuSZp-cpu+V5pack", + String.valueOf(encMb2), + String.valueOf(decMb2), + String.valueOf((long) points), + String.valueOf((long) avgBits2), + String.valueOf(ratio2) + }); + } finally { + w2.close(); + } + System.out.println("debug one CSV V5: " + file.getName() + " -> " + outCsv2 + " encMB/s=" + encMb2); + } + + /** + * CuSZp2 baseline (fixed pack size 8). Writes same CSV schema as + * {@code fig_alp_cuszp_pack8_vs_v5_combined.py} input dir {@code OUTPUT_CUSZP_DIR} / {@code output_cuszp_cpu}. + */ @Test public void cuSZpCpu1DTest() throws Exception { System.out.println("\nCPU cuSZp-like plain-mode Performance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_cuszp_cpu"; + String directory = resolveDataDir().getAbsolutePath(); + String outputDirstr = new File(resolveOutputBaseDir(), "output_cuszp_cpu").getAbsolutePath(); File outputDir = new File(outputDirstr); if (!outputDir.exists()) outputDir.mkdir(); @@ -70,7 +336,7 @@ public void cuSZpCpu1DTest() throws Exception { } // parameters - double eb = 1e-4; // absolute error bound, 可让用户传参 / 读取 + double eb = resolveBenchEb(); int packSize = 8; // how many values per pack when computing bitwidth int repeats = 20; // average times @@ -135,11 +401,15 @@ public void cuSZpCpu1DTest() throws Exception { } } + /** + * CuSZp2 + optimal pack (Prune-RMQ / V5). Writes same CSV schema as combined figure + * {@code OUTPUT_CUSZP_V5_DIR} / {@code output_cuszp_cpu_optimal_v5}. + */ @Test public void cuSZpCpu1DOptimalV5Test() throws Exception { System.out.println("\nCPU cuSZp-like plain-mode (optimal pack V5 per chunk)..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_cuszp_cpu_optimal_v5"; + String directory = resolveDataDir().getAbsolutePath(); + String outputDirstr = new File(resolveOutputBaseDir(), "output_cuszp_cpu_optimal_v5").getAbsolutePath(); File outputDir = new File(outputDirstr); if (!outputDir.exists()) outputDir.mkdir(); @@ -172,7 +442,7 @@ public void cuSZpCpu1DOptimalV5Test() throws Exception { continue; } - double eb = 1e-4; + double eb = resolveBenchEb(); int repeats = 20; long totalEncodeNs = 0; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java index efdd5c5f2..cc2445622 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLong.java @@ -1,6 +1,8 @@ package org.apache.tsfile.encoding; -import java.util.*; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Objects; public class HBPIndexLong { @@ -47,6 +49,58 @@ public HBPIndexLong(int kBits, long[] codes) { pack(codes); } + /** + * Number of {@code long} words in the packed representation for {@code nCodes} codes at bit width {@code kBits} + * (same layout as {@link #pack(long[])}). + */ + public static int packedLongCount(int kBits, int nCodes) { + int sectionBits = kBits + 1; + int sectionsPerWord = W / sectionBits; + if (sectionsPerWord <= 0) { + throw new IllegalArgumentException("k too large for 64-bit word"); + } + int codesPerSegment = sectionsPerWord * (kBits + 1); + int segments = (nCodes + codesPerSegment - 1) / codesPerSegment; + return segments * (kBits + 1); + } + + /** + * Reconstruct an index from packed {@code long} words (e.g. after deserializing from a byte stream), without + * re-running {@link #pack(long[])}. + */ + public static HBPIndexLong fromPackedWords(int kBits, int nCodes, long[] packedWords) { + Objects.requireNonNull(packedWords, "packedWords"); + int expected = packedLongCount(kBits, nCodes); + if (packedWords.length != expected) { + throw new IllegalArgumentException( + "packedWords.length mismatch: expected " + expected + " for k=" + kBits + ", n=" + nCodes + + " but got " + packedWords.length); + } + return new HBPIndexLong(kBits, nCodes, packedWords); + } + + private HBPIndexLong(int kBits, int nCodes, long[] packedWords) { + this.k = kBits; + this.sectionBits = kBits + 1; + this.sectionsPerWord = W / sectionBits; + if (sectionsPerWord <= 0) { + throw new IllegalArgumentException("k too large for 64-bit word"); + } + this.codesPerSegment = sectionsPerWord * (kBits + 1); + this.n = nCodes; + this.segments = (nCodes + codesPerSegment - 1) / codesPerSegment; + int expectedWords = segments * (kBits + 1); + if (packedWords.length != expectedWords) { + throw new IllegalArgumentException( + "packedWords.length mismatch: expected " + expectedWords + " for k=" + kBits + ", n=" + nCodes + + " but got " + packedWords.length); + } + this.words = Arrays.copyOf(packedWords, packedWords.length); + this.lowKOnesRepeat = repeatInSections((1L << kBits) - 1L); + this.delimiterBitRepeat = repeatInSections(1L << kBits); + this.addOneEachSection = repeatInSections(1L); + } + public BitSet select(Op op, long C) { return selectInternal(op, C & ((1L << k) - 1)); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java index d659ec0a1..abec20364 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/HBPIndexLongTest.java @@ -100,12 +100,23 @@ public static int BlockEncoder(long[] data, int block_index, int block_size, int HBPIndexLong idx = new HBPIndexLong(bw, block_data); indexList.add(idx); + // Encode words with BP-style BitWriterV2 (32+32 MSB-first per long); decode uses bytes2Long (same wire layout). + AllNo8PacksizeOptimal.BitWriterV2 bitWriter = new AllNo8PacksizeOptimal.BitWriterV2(); + for (int w = 0; w < idx.words.length; w++) { + long word = idx.words[w]; + bitWriter.writeBits((int) (word >>> 32), 32); + bitWriter.writeBits((int) (word & 0xFFFFFFFFL), 32); + } + byte[] wordPayload = bitWriter.toByteArray(); + System.arraycopy(wordPayload, 0, encoded_result, encode_pos, wordPayload.length); + encode_pos += wordPayload.length; + return encode_pos; } public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder, - int encode_pos, ArrayList indexList, long[] data) { + int encode_pos, long[] data) { long min_value = bytes2Long(encoded_result, encode_pos, 8); encode_pos += 8; @@ -113,7 +124,14 @@ public static int BlockDecoder(byte[] encoded_result, int block_index, int block int bw = bytes2Integer(encoded_result, encode_pos, 4); encode_pos += 4; - HBPIndexLong idx = indexList.get(block_index); + int nw = HBPIndexLong.packedLongCount(bw, remainder); + // BitWriterV2 emits MSB-first 32+32 bits per long → same 8-byte layout as long2Bytes / bytes2Long. + long[] words = new long[nw]; + for (int w = 0; w < nw; w++) { + words[w] = bytes2Long(encoded_result, encode_pos, 8); + encode_pos += Long.BYTES; + } + HBPIndexLong idx = HBPIndexLong.fromPackedWords(bw, remainder, words); for (int i = 0; i < remainder; i++) { long value = idx.getCode(i); @@ -149,7 +167,8 @@ public static int Encoder(long[] data, int block_size, ArrayList i return encode_pos; } - public static long[] Decoder(byte[] encoded_result, ArrayList indexList) { + /** Full decode from {@link #Encoder}: rebuilds {@link HBPIndexLong} from serialized bytes each call. */ + public static long[] Decoder(byte[] encoded_result) { int encode_pos = 0; int data_length = bytes2Integer(encoded_result, encode_pos, 4); @@ -163,13 +182,13 @@ public static long[] Decoder(byte[] encoded_result, ArrayList inde long[] data = new long[data_length]; for (int i = 0; i < num_blocks; i++) { - encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, data); + encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data); } int remainder = data_length % block_size; encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, - encode_pos, indexList, data); + encode_pos, data); return data; } @@ -205,8 +224,8 @@ public static String extractFileName(String path) { public void test0() throws IOException { String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; String outputDirStr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_Bitweaving"; - int block_size = 512; - int repeatTime = 50; + int block_size = 1024; + int repeatTime = 500; File dir = new File(directory); Assume.assumeTrue( @@ -262,7 +281,8 @@ public void test0() throws IOException { data2_arr[i] = (long) (data1.get(i) * max_mul); } - byte[] encoded_result = new byte[data2_arr.length * 8]; + // Header + per-block (min + bw + packed long words); 2× raw long payload is conservative. + byte[] encoded_result = new byte[8 + data2_arr.length * Long.BYTES * 2]; ArrayList indexList = new ArrayList<>(); long s = System.nanoTime(); @@ -274,17 +294,15 @@ public void test0() throws IOException { long e = System.nanoTime(); long encodeTimeNs = (e - s) / repeatTime; + // Serialized size: file header + each block's min, bw, and packed words (same as on-the-wire). long compressedBytes = length; - for (HBPIndexLong idx : indexList) { - compressedBytes += idx.segments * (idx.k + 1) * Long.BYTES; - } long compressedSizeBits = compressedBytes * 8L; double ratio = (double) compressedSizeBits / (double) (data1.size() * 64L); long[] data2_arr_decoded = new long[data2_arr.length]; s = System.nanoTime(); for (int repeat = 0; repeat < repeatTime; repeat++) { - data2_arr_decoded = Decoder(encoded_result, indexList); + data2_arr_decoded = Decoder(encoded_result); } e = System.nanoTime(); long decodeTimeNs = (e - s) / repeatTime; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/MemoryTsFileOutput.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/MemoryTsFileOutput.java new file mode 100644 index 000000000..463083be3 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/MemoryTsFileOutput.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding; + +import org.apache.tsfile.write.writer.TsFileOutput; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** In-memory {@link TsFileOutput} for benchmarking encode time without disk I/O. */ +public final class MemoryTsFileOutput implements TsFileOutput { + + private byte[] buf = new byte[4 * 1024 * 1024]; + private int len = 0; + + private void ensure(int add) { + int need = len + add; + if (need <= buf.length) { + return; + } + int newCap = buf.length; + while (newCap < need) { + newCap = newCap < (1 << 30) ? newCap << 1 : need; + } + buf = Arrays.copyOf(buf, newCap); + } + + @Override + public void write(byte[] b) throws IOException { + ensure(b.length); + System.arraycopy(b, 0, buf, len, b.length); + len += b.length; + } + + @Override + public void write(byte b) throws IOException { + ensure(1); + buf[len++] = b; + } + + @Override + public void write(ByteBuffer b) throws IOException { + int n = b.remaining(); + ensure(n); + if (b.hasArray()) { + System.arraycopy(b.array(), b.arrayOffset() + b.position(), buf, len, n); + len += n; + b.position(b.limit()); + } else { + for (int i = 0; i < n; i++) { + buf[len++] = b.get(); + } + } + } + + @Override + public long getPosition() { + return len; + } + + @Override + public void close() { + // no-op + } + + @Override + public OutputStream wrapAsStream() { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + MemoryTsFileOutput.this.write((byte) b); + } + + @Override + public void write(byte[] b, int off, int ln) throws IOException { + ensure(ln); + System.arraycopy(b, off, buf, len, ln); + len += ln; + } + }; + } + + @Override + public void flush() { + // no-op + } + + @Override + public void truncate(long size) throws IOException { + if (size < 0 || size > Integer.MAX_VALUE) { + throw new IOException("invalid truncate size: " + size); + } + len = (int) size; + } + + @Override + public void force() { + // no-op (no disk) + } + + public byte[] toByteArray() { + return Arrays.copyOf(buf, len); + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/TsFilePerPageDiskIoHelper.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/TsFilePerPageDiskIoHelper.java new file mode 100644 index 000000000..dbb2a6cd4 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/TsFilePerPageDiskIoHelper.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.encoding; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.reader.TsFileInput; +import org.apache.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +/** + * Splits a TsFile byte array into consecutive segments so that each page inside a chunk + * body is its own segment (chunk headers and non-chunk gaps are separate segments). Used to time + * disk I/O as one physical write/read per segment. + */ +public final class TsFilePerPageDiskIoHelper { + + private TsFilePerPageDiskIoHelper() {} + + /** + * Build non-overlapping {@code [offset, length)} segments covering the whole file, ordered by + * offset. Chunk data regions are split per page; other bytes are grouped into gap/header + * segments. + * + *

Walks the data section in on-disk order using the same marker/page rules as {@link + * TsFileSequenceReader#selfCheck}, so results stay consistent with {@link ByteArrayTsFileInput} + * (see {@link TsFileInput#wrapAsInputStream()} position sync). + */ + public static List buildContiguousSegments(byte[] tsfileBytes) throws IOException { + Objects.requireNonNull(tsfileBytes, "tsfileBytes"); + int fileLen = tsfileBytes.length; + int headerLength = TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; + + ByteArrayTsFileInput input = new ByteArrayTsFileInput(tsfileBytes); + TsFileSequenceReader reader = new TsFileSequenceReader(input); + try { + reader.readFileMetadata(); + input.position(headerLength); + + List segments = new ArrayList<>(); + segments.add(new int[] {0, headerLength}); + + while (true) { + long markPos = input.position(); + byte marker = reader.readMarker(); + if (marker == MetaMarker.SEPARATOR) { + segments.add(new int[] {(int) markPos, fileLen - (int) markPos}); + break; + } + switch (marker) { + case MetaMarker.CHUNK_HEADER: + case MetaMarker.TIME_CHUNK_HEADER: + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + { + long chunkStart = markPos; + ChunkHeader chunkHeader = reader.readChunkHeader(marker); + int headerSize = chunkHeader.getSerializedSize(); + segments.add(new int[] {(int) chunkStart, headerSize}); + + int dataSize = chunkHeader.getDataSize(); + if (dataSize > 0) { + if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.CHUNK_HEADER) { + while (dataSize > 0) { + long pageStart = input.position(); + PageHeader pageHeader = + reader.readPageHeader(chunkHeader.getDataType(), true); + if (pageHeader.getUncompressedSize() != 0) { + reader.skipPageData(pageHeader); + } + long pageEnd = input.position(); + segments.add( + new int[] {(int) pageStart, (int) (pageEnd - pageStart)}); + dataSize -= pageHeader.getSerializedPageSize(); + } + } else { + long pageStart = input.position(); + PageHeader pageHeader = + reader.readPageHeader(chunkHeader.getDataType(), false); + if (pageHeader.getUncompressedSize() != 0) { + reader.skipPageData(pageHeader); + } + long pageEnd = input.position(); + segments.add( + new int[] {(int) pageStart, (int) (pageEnd - pageStart)}); + } + } + break; + } + case MetaMarker.CHUNK_GROUP_HEADER: + { + long groupStart = markPos; + reader.readChunkGroupHeader(); + long groupEnd = input.position(); + segments.add(new int[] {(int) groupStart, (int) (groupEnd - groupStart)}); + break; + } + case MetaMarker.OPERATION_INDEX_RANGE: + { + long opStart = markPos; + reader.readPlanIndex(); + long opEnd = input.position(); + segments.add(new int[] {(int) opStart, (int) (opEnd - opStart)}); + break; + } + default: + throw new IOException("Unexpected marker " + marker + " at offset " + markPos); + } + } + + return mergeGapsAndVerify(segments, fileLen); + } finally { + reader.close(); + } + } + + /** Write each segment with a separate timed {@link FileChannel#write(ByteBuffer, long)} loop. */ + public static long writeAllSegmentsTimed(File outFile, byte[] data, List segments) + throws IOException { + long totalNs = 0; + try (FileChannel ch = + FileChannel.open( + outFile.toPath(), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + for (int[] seg : segments) { + ByteBuffer bb = ByteBuffer.wrap(data, seg[0], seg[1]); + long pos = seg[0]; + long t0 = System.nanoTime(); + while (bb.hasRemaining()) { + int w = ch.write(bb, pos); + if (w <= 0) { + throw new IOException("FileChannel.write stalled"); + } + pos += w; + } + totalNs += System.nanoTime() - t0; + } + } + return totalNs; + } + + /** Read each segment with a separate timed {@link RandomAccessFile#readFully(byte[], int, int)}. */ + public static Pair readAllSegmentsTimed(File tsfile, List segments) + throws IOException { + int len = (int) tsfile.length(); + byte[] buf = new byte[len]; + long totalNs = 0; + try (RandomAccessFile raf = new RandomAccessFile(tsfile, "r")) { + for (int[] seg : segments) { + long t0 = System.nanoTime(); + raf.seek(seg[0]); + raf.readFully(buf, seg[0], seg[1]); + totalNs += System.nanoTime() - t0; + } + } + return new Pair<>(buf, totalNs); + } + + /** + * Sort segments, drop empty spans, insert gap segments for any hole, append tail to EOF, and + * verify full coverage. Fixes missing header/page slices when metadata or page parsing is + * slightly inconsistent. + */ + private static List mergeGapsAndVerify(List segments, int fileLen) throws IOException { + segments.removeIf(s -> s[1] <= 0); + segments.sort(Comparator.comparingInt(a -> a[0])); + List merged = new ArrayList<>(); + int expect = 0; + for (int[] s : segments) { + if (s[0] < expect) { + throw new IOException("Overlapping segments at " + s[0] + ", expected end " + expect); + } + if (s[0] > expect) { + merged.add(new int[] {expect, s[0] - expect}); + } + merged.add(s); + expect = s[0] + s[1]; + } + if (expect < fileLen) { + merged.add(new int[] {expect, fileLen - expect}); + } + if (expect > fileLen) { + throw new IOException("Segments extend past EOF: end " + expect + " fileLen " + fileLen); + } + return merged; + } +} From 9bac282a64ea9bd1bf3e5bed1c94e4d9c776c14a Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:20:40 +0800 Subject: [PATCH 5/9] Update AllNo8PacksizeOptimal.java --- .../encoding/AllNo8PacksizeOptimal.java | 115 +++++------------- 1 file changed, 30 insertions(+), 85 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java index 5cd70c3b0..9ed41b2d9 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -54,7 +54,7 @@ public class AllNo8PacksizeOptimal { * {@code startEncodeTime}/{@code encodeDuration} and {@code startDecodeTime}/{@code decodeDuration}, * see {@link #benchChunkedBitPacking} and {@link #BPTest()}). */ - private static final int DEFAULT_BENCH_TIME_REPEAT = 100; + private static final int DEFAULT_BENCH_TIME_REPEAT = 400; /** One chunk's packed payload for repeated decode timing. */ private static final class EncodedChunk { @@ -5881,7 +5881,7 @@ public void VaryPackSizeTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); - int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -6217,7 +6217,7 @@ public void VaryPackSizeSprintzTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); - int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -6261,7 +6261,7 @@ public void VaryPackSizeSprintzTest() throws IOException { csvReader.close(); int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - int time_of_repeat = 50; + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; // 缩放数据 int batchSize = 1024; @@ -6282,98 +6282,43 @@ public void VaryPackSizeSprintzTest() throws IOException { currentIndex += batch.length; } - // 测试每个pack size + // 测试每个 pack size（与 OptimalPackSizeRMQSprintzSortTest：benchChunkedBitPacking） for (int packSize : packSizes) { System.out.println("Testing pack size: " + packSize); - long totalEncodeTime = 0; - long totalDecodeTime = 0; - long totalCompressedSize = 0; - int totalPoints = 0; - - for (int j = 0; j < time_of_repeat; j++) { - for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { - int end = Math.min(i + CHUNK_SIZE, numbers.size()); - int[] scaledInt = new int[end - i]; - if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); - - // 编码 - long startEncodeTime = System.nanoTime(); - int[] scaledInts = sprintz(scaledInt); - - // 计算需要的组数 - int numGroups = (scaledInts.length + packSize - 1) / packSize; - int[] bitWidths = new int[numGroups]; - - // 计算每个组的位宽 - for (int group = 0; group < numGroups; group++) { - int startIdx = group * packSize; - int endIdx = Math.min(startIdx + packSize, scaledInts.length); - - int maxInGroup = 0; - for (int idx = startIdx; idx < endIdx; idx++) { - if (scaledInts[idx] > maxInGroup) { - maxInGroup = scaledInts[idx]; - } - } - - int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); - bitWidths[group] = bitWidth; - } - - // 编码数据 - byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); - long encodeDuration = System.nanoTime() - startEncodeTime; - - // 解码 - long startDecodeTime = System.nanoTime(); - int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); - int[] sprintzDecoded = sprintzDecode(decodedData); - long decodeDuration = System.nanoTime() - startDecodeTime; - -// // 验证解码结果 -// boolean valid = true; -// for (int k = 0; k < scaledInts.length; k++) { -// if (scaledInts[k] != decodedData[k]) { -// System.err.println("Decode error at position " + k + -// ": expected " + scaledInts[k] + ", got " + decodedData[k]); -// valid = false; -// break; -// } -// } -// -// if (!valid) { -// System.err.println("Decoding failed for pack size " + packSize); -// } - - // 累加统计 - totalEncodeTime += encodeDuration; - totalDecodeTime += decodeDuration; - totalCompressedSize += compressedData.length * 8L; // 转换为bits - totalPoints += scaledInts.length; - } - } - - // 计算平均 - long avgEncodeTime = totalEncodeTime / time_of_repeat; - long avgDecodeTime = totalDecodeTime / time_of_repeat; - long avgCompressedSize = totalCompressedSize / time_of_repeat; - totalPoints /= time_of_repeat; + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + final int ps = Integer.max(1, packSize); + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunkArg -> { + int[] scaledInts = sprintz(chunkArg); + return encodeChunkBitPacking(scaledInts, ps); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; - // 计算吞吐率和压缩率 - double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s - double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s - double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + double compressionRatio = (double) modelCost / (double) (numbers.size() * 64); + double encodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double decodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); - // 写入结果 String[] record = { file.toString(), "Sprintz", String.valueOf(encodeThroughput), String.valueOf(decodeThroughput), - String.valueOf(totalPoints), + String.valueOf(numbers.size()), String.valueOf(packSize), - String.valueOf(avgCompressedSize), + String.valueOf(modelCost), String.valueOf(compressionRatio) }; writer.writeRecord(record); From ccc783ab95e1607ce5b60bec1e388cf5f7e3ca6a Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:31:24 +0800 Subject: [PATCH 6/9] Update AllNo8PacksizeOptimal.java --- .../encoding/AllNo8PacksizeOptimal.java | 121 +++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java index 9ed41b2d9..f78de609e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -54,7 +54,7 @@ public class AllNo8PacksizeOptimal { * {@code startEncodeTime}/{@code encodeDuration} and {@code startDecodeTime}/{@code decodeDuration}, * see {@link #benchChunkedBitPacking} and {@link #BPTest()}). */ - private static final int DEFAULT_BENCH_TIME_REPEAT = 400; + private static final int DEFAULT_BENCH_TIME_REPEAT = 100; /** One chunk's packed payload for repeated decode timing. */ private static final class EncodedChunk { @@ -4198,6 +4198,125 @@ public void BPTest() throws IOException { } } + + @Test + public void SprintzTest() throws IOException { + System.out.println("\nPerformance Testing..."); + String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; + String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_sprintz"; + File outputDir = new File(outputDirstr); + + if (!outputDir.exists()) outputDir.mkdir(); + File dir = new File(directory); + Assume.assumeTrue( + "Skip BPTest: dataset directory missing: " + directory, + dir.exists() && dir.isDirectory() + ); + for (File file : Objects.requireNonNull(dir.listFiles())) { + + if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; + System.out.println(file.getName()); + String Output = outputDirstr + "/" + file.getName(); + CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); + + String[] head = { + "Input Direction", + "Encoding Algorithm", + "Encoding Time", + "Decoding Time", + "Points", + "Compressed Size", + "Compression Ratio" + }; + writer.writeRecord(head); + System.out.println("Processing " + file.getName() + "..."); + List numbers = new ArrayList<>(); + List decimalPlaces = new ArrayList<>(); + CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); + while (csvReader.readRecord()) { + for (String value : csvReader.getValues()) { + String numStr = value.trim(); + if (!numStr.isEmpty()) { + numbers.add(numStr); + int decimal = 0, sigBits; + if (numStr.contains(".")) { + String[] parts = numStr.split("\\."); + decimal = parts[1].length(); + sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); + } else { + sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); + } + decimalPlaces.add(decimal); + } + } + } + int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; + + int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); + + // 分批处理，每1024个元素一批 + int batchSize = 1024; + List batches = new ArrayList<>(); + + for (int i = 0; i < numbers.size(); i += batchSize) { + int end = Math.min(numbers.size(), i + batchSize); + List batch = numbers.subList(i, end); + int[] scaledBatch = scaleNumbers(batch, decimalMax); + batches.add(scaledBatch); + } + + // 计算总长度并拼接所有批次的结果 + int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); + int[] scaledInts_all = new int[totalLength]; + + int currentIndex = 0; + for (int[] batch : batches) { + System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); + currentIndex += batch.length; + } + long[] costA = new long[1]; + long[] encA = new long[1]; + long[] decA = new long[1]; + benchChunkedBitPacking( + scaledInts_all, + numbers.size(), + CHUNK_SIZE, + time_of_repeat, + chunk -> { + int[] scaledInts = sprintz(chunk); + return encodeChunkBitPacking(scaledInts, 8); + }, + ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), + costA, + encA, + decA); + long modelCost = costA[0]; + long modelTime = encA[0]; + long modelDecodeTime = decA[0]; + + double model_ratio = (double) modelCost / (double) (numbers.size() * 64); + double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); + double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + + String[] record = { + file.toString(), + "BP", + String.valueOf(modelTime_throughput), + String.valueOf(modelDecodeTime_throughput), + String.valueOf(numbers.size()), + String.valueOf(modelCost), + String.valueOf(model_ratio) + }; + writer.writeRecord(record); + writer.close(); + + System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); + System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); + System.out.println("Compression ratio: " + model_ratio); + } + } + + @Test public void Simple8bTest() throws IOException { System.out.println("\nPerformance Testing (Simple8b)..."); From cbfafdadeeafc57210d4dff7f13d0238d38b0f92 Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Sun, 5 Apr 2026 13:31:21 +0800 Subject: [PATCH 7/9] Update AllNo8PacksizeOptimal.java --- .../encoding/AllNo8PacksizeOptimal.java | 340 +++++++++--------- 1 file changed, 166 insertions(+), 174 deletions(-) diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java index f78de609e..e8a422ea0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/AllNo8PacksizeOptimal.java @@ -4198,125 +4198,6 @@ public void BPTest() throws IOException { } } - - @Test - public void SprintzTest() throws IOException { - System.out.println("\nPerformance Testing..."); - String directory = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/ElfTestData_camel"; - String outputDirstr = "/Users/xiaojinzhao/Documents/GitHub/encoding-pack-size/output_sprintz"; - File outputDir = new File(outputDirstr); - - if (!outputDir.exists()) outputDir.mkdir(); - File dir = new File(directory); - Assume.assumeTrue( - "Skip BPTest: dataset directory missing: " + directory, - dir.exists() && dir.isDirectory() - ); - for (File file : Objects.requireNonNull(dir.listFiles())) { - - if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; - System.out.println(file.getName()); - String Output = outputDirstr + "/" + file.getName(); - CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8); - - String[] head = { - "Input Direction", - "Encoding Algorithm", - "Encoding Time", - "Decoding Time", - "Points", - "Compressed Size", - "Compression Ratio" - }; - writer.writeRecord(head); - System.out.println("Processing " + file.getName() + "..."); - List numbers = new ArrayList<>(); - List decimalPlaces = new ArrayList<>(); - CsvReader csvReader = new CsvReader(file.getPath(), ',', StandardCharsets.UTF_8); - while (csvReader.readRecord()) { - for (String value : csvReader.getValues()) { - String numStr = value.trim(); - if (!numStr.isEmpty()) { - numbers.add(numStr); - int decimal = 0, sigBits; - if (numStr.contains(".")) { - String[] parts = numStr.split("\\."); - decimal = parts[1].length(); - sigBits = (int) ((parts[0].length() + decimal) * (Math.log(10) / Math.log(2))); - } else { - sigBits = (int) (numStr.length() * (Math.log(10) / Math.log(2))); - } - decimalPlaces.add(decimal); - } - } - } - int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; - - int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - - // 分批处理，每1024个元素一批 - int batchSize = 1024; - List batches = new ArrayList<>(); - - for (int i = 0; i < numbers.size(); i += batchSize) { - int end = Math.min(numbers.size(), i + batchSize); - List batch = numbers.subList(i, end); - int[] scaledBatch = scaleNumbers(batch, decimalMax); - batches.add(scaledBatch); - } - - // 计算总长度并拼接所有批次的结果 - int totalLength = batches.stream().mapToInt(arr -> arr.length).sum(); - int[] scaledInts_all = new int[totalLength]; - - int currentIndex = 0; - for (int[] batch : batches) { - System.arraycopy(batch, 0, scaledInts_all, currentIndex, batch.length); - currentIndex += batch.length; - } - long[] costA = new long[1]; - long[] encA = new long[1]; - long[] decA = new long[1]; - benchChunkedBitPacking( - scaledInts_all, - numbers.size(), - CHUNK_SIZE, - time_of_repeat, - chunk -> { - int[] scaledInts = sprintz(chunk); - return encodeChunkBitPacking(scaledInts, 8); - }, - ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), - costA, - encA, - decA); - long modelCost = costA[0]; - long modelTime = encA[0]; - long modelDecodeTime = decA[0]; - - double model_ratio = (double) modelCost / (double) (numbers.size() * 64); - double modelTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelTime); - double modelDecodeTime_throughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); - - String[] record = { - file.toString(), - "BP", - String.valueOf(modelTime_throughput), - String.valueOf(modelDecodeTime_throughput), - String.valueOf(numbers.size()), - String.valueOf(modelCost), - String.valueOf(model_ratio) - }; - writer.writeRecord(record); - writer.close(); - - System.out.println("Optimal pack_size found, encoding throughput: " + modelTime_throughput + " MB/s"); - System.out.println("Decoding throughput: " + modelDecodeTime_throughput + " MB/s"); - System.out.println("Compression ratio: " + model_ratio); - } - } - - @Test public void Simple8bTest() throws IOException { System.out.println("\nPerformance Testing (Simple8b)..."); @@ -6000,7 +5881,7 @@ public void VaryPackSizeTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); - int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -6044,7 +5925,7 @@ public void VaryPackSizeTest() throws IOException { csvReader.close(); int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; + int time_of_repeat = 50; // 缩放数据 int batchSize = 1024; @@ -6065,40 +5946,96 @@ public void VaryPackSizeTest() throws IOException { currentIndex += batch.length; } - // 测试每个 pack size（与 OptimalPackSizeRMQSprintzSortTest：benchChunkedBitPacking） + // 测试每个pack size for (int packSize : packSizes) { System.out.println("Testing pack size: " + packSize); - long[] costA = new long[1]; - long[] encA = new long[1]; - long[] decA = new long[1]; - final int ps = Integer.max(1, packSize); - benchChunkedBitPacking( - scaledInts_all, - numbers.size(), - CHUNK_SIZE, - time_of_repeat, - chunk -> encodeChunkBitPacking(chunk, ps), - ec -> decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts), - costA, - encA, - decA); - long modelCost = costA[0]; - long modelTime = encA[0]; - long modelDecodeTime = decA[0]; - - double compressionRatio = (double) modelCost / (double) (numbers.size() * 64); - double encodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelTime); - double decodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInts = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInts, 0, end - i); + + // 编码 + long startEncodeTime = System.nanoTime(); + + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + } + } + + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + + // 写入结果 String[] record = { file.toString(), "BitPacking", String.valueOf(encodeThroughput), String.valueOf(decodeThroughput), - String.valueOf(numbers.size()), + String.valueOf(totalPoints), String.valueOf(packSize), - String.valueOf(modelCost), + String.valueOf(avgCompressedSize), String.valueOf(compressionRatio) }; writer.writeRecord(record); @@ -6336,7 +6273,7 @@ public void VaryPackSizeSprintzTest() throws IOException { if (!outputDir.exists()) outputDir.mkdir(); File dir = new File(directory); - int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}; + int[] packSizes = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048}; for (File file : Objects.requireNonNull(dir.listFiles())) { if (IGNORE_FILES.contains(file.getName()) || file.isDirectory()) continue; @@ -6380,7 +6317,7 @@ public void VaryPackSizeSprintzTest() throws IOException { csvReader.close(); int decimalMax = decimalPlaces.stream().max(Integer::compare).orElse(0); - int time_of_repeat = DEFAULT_BENCH_TIME_REPEAT; + int time_of_repeat = 50; // 缩放数据 int batchSize = 1024; @@ -6401,43 +6338,98 @@ public void VaryPackSizeSprintzTest() throws IOException { currentIndex += batch.length; } - // 测试每个 pack size（与 OptimalPackSizeRMQSprintzSortTest：benchChunkedBitPacking） + // 测试每个pack size for (int packSize : packSizes) { System.out.println("Testing pack size: " + packSize); - long[] costA = new long[1]; - long[] encA = new long[1]; - long[] decA = new long[1]; - final int ps = Integer.max(1, packSize); - benchChunkedBitPacking( - scaledInts_all, - numbers.size(), - CHUNK_SIZE, - time_of_repeat, - chunkArg -> { - int[] scaledInts = sprintz(chunkArg); - return encodeChunkBitPacking(scaledInts, ps); - }, - ec -> sprintzDecode(decodeBitPackingV2(ec.compressed, ec.bitWidths, ec.packSize, ec.nInts)), - costA, - encA, - decA); - long modelCost = costA[0]; - long modelTime = encA[0]; - long modelDecodeTime = decA[0]; - - double compressionRatio = (double) modelCost / (double) (numbers.size() * 64); - double encodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelTime); - double decodeThroughput = (double) (numbers.size() * 8000L) / (double) (modelDecodeTime); + long totalEncodeTime = 0; + long totalDecodeTime = 0; + long totalCompressedSize = 0; + int totalPoints = 0; + + for (int j = 0; j < time_of_repeat; j++) { + for (int i = 0; i < numbers.size(); i += CHUNK_SIZE) { + int end = Math.min(i + CHUNK_SIZE, numbers.size()); + int[] scaledInt = new int[end - i]; + if (end - i >= 0) System.arraycopy(scaledInts_all, i, scaledInt, 0, end - i); + // 编码 + long startEncodeTime = System.nanoTime(); + int[] scaledInts = sprintz(scaledInt); + + // 计算需要的组数 + int numGroups = (scaledInts.length + packSize - 1) / packSize; + int[] bitWidths = new int[numGroups]; + + // 计算每个组的位宽 + for (int group = 0; group < numGroups; group++) { + int startIdx = group * packSize; + int endIdx = Math.min(startIdx + packSize, scaledInts.length); + + int maxInGroup = 0; + for (int idx = startIdx; idx < endIdx; idx++) { + if (scaledInts[idx] > maxInGroup) { + maxInGroup = scaledInts[idx]; + } + } + + int bitWidth = 64 - Long.numberOfLeadingZeros(Math.max(1, maxInGroup)); + bitWidths[group] = bitWidth; + } + + // 编码数据 + byte[] compressedData = encodeBitPackingV2(scaledInts, bitWidths, packSize); + long encodeDuration = System.nanoTime() - startEncodeTime; + + // 解码 + long startDecodeTime = System.nanoTime(); + int[] decodedData = decodeBitPackingV2(compressedData, bitWidths, packSize, scaledInts.length); + int[] sprintzDecoded = sprintzDecode(decodedData); + long decodeDuration = System.nanoTime() - startDecodeTime; + +// // 验证解码结果 +// boolean valid = true; +// for (int k = 0; k < scaledInts.length; k++) { +// if (scaledInts[k] != decodedData[k]) { +// System.err.println("Decode error at position " + k + +// ": expected " + scaledInts[k] + ", got " + decodedData[k]); +// valid = false; +// break; +// } +// } +// +// if (!valid) { +// System.err.println("Decoding failed for pack size " + packSize); +// } + + // 累加统计 + totalEncodeTime += encodeDuration; + totalDecodeTime += decodeDuration; + totalCompressedSize += compressedData.length * 8L; // 转换为bits + totalPoints += scaledInts.length; + } + } + + // 计算平均 + long avgEncodeTime = totalEncodeTime / time_of_repeat; + long avgDecodeTime = totalDecodeTime / time_of_repeat; + long avgCompressedSize = totalCompressedSize / time_of_repeat; + totalPoints /= time_of_repeat; + + // 计算吞吐率和压缩率 + double encodeThroughput = (double) (totalPoints * 8000L) / (double) avgEncodeTime; // MB/s + double decodeThroughput = (double) (totalPoints * 8000L) / (double) avgDecodeTime; // MB/s + double compressionRatio = (double) avgCompressedSize / (double) (totalPoints * 64); + + // 写入结果 String[] record = { file.toString(), "Sprintz", String.valueOf(encodeThroughput), String.valueOf(decodeThroughput), - String.valueOf(numbers.size()), + String.valueOf(totalPoints), String.valueOf(packSize), - String.valueOf(modelCost), + String.valueOf(avgCompressedSize), String.valueOf(compressionRatio) }; writer.writeRecord(record); From c05c1425bc6c21cab4a6ca2ab74276b8b5d11403 Mon Sep 17 00:00:00 2001 From: xjz17 <67282793+xjz17@users.noreply.github.com> Date: Thu, 7 May 2026 09:33:57 +0800 Subject: [PATCH 8/9] update float lossless --- cpp/third_party/zlib-1.3.1/treebuild.xml | 188 +- .../zlib-1.3.1/zlib-1.3.1/treebuild.xml | 188 +- .../org/apache/tsfile/encoding/ALPTest.java | 28 +- .../encoding/AllNo8PacksizeOptimal.java | 949 +- .../apache/tsfile/encoding/CuSZpCpuTest.java | 2 +- .../test/resources/TestData/Air-pressure.csv | 100001 ++ .../resources/TestData/Bird-migration.csv | 89867 ++ .../test/resources/TestData/Blockchain-tr.csv | 99999 ++ .../src/test/resources/TestData/City-lat.csv | 41001 + .../src/test/resources/TestData/City-lon.csv | 41001 + .../src/test/resources/TestData/City-temp.csv | 100001 ++ .../test/resources/TestData/Cyber-Vehicle.csv | 7281 + .../resources/TestData/Dew-point-temp.csv | 100001 ++ .../test/resources/TestData/Food-price.csv | 1048575 ++++++++++++++ .../test/resources/TestData/IR-bio-temp.csv | 100000 ++ .../src/test/resources/TestData/PM10-dust.csv | 100002 ++ .../src/test/resources/TestData/Stocks-DE.csv | 100002 ++ .../src/test/resources/TestData/Stocks-UK.csv | 100002 ++ .../test/resources/TestData/Stocks-USA.csv | 100002 ++ .../src/test/resources/TestData/TY-Fuel.csv | 58532 + .../test/resources/TestData/TY-Transport.csv | 51223 + .../test/resources/TestData/Wind-Speed.csv | 100000 ++ 22 files changed, 2338222 insertions(+), 623 deletions(-) create mode 100644 java/tsfile/src/test/resources/TestData/Air-pressure.csv create mode 100644 java/tsfile/src/test/resources/TestData/Bird-migration.csv create mode 100644 java/tsfile/src/test/resources/TestData/Blockchain-tr.csv create mode 100644 java/tsfile/src/test/resources/TestData/City-lat.csv create mode 100644 java/tsfile/src/test/resources/TestData/City-lon.csv create mode 100644 java/tsfile/src/test/resources/TestData/City-temp.csv create mode 100644 java/tsfile/src/test/resources/TestData/Cyber-Vehicle.csv create mode 100644 java/tsfile/src/test/resources/TestData/Dew-point-temp.csv create mode 100644 java/tsfile/src/test/resources/TestData/Food-price.csv create mode 100644 java/tsfile/src/test/resources/TestData/IR-bio-temp.csv create mode 100644 java/tsfile/src/test/resources/TestData/PM10-dust.csv create mode 100644 java/tsfile/src/test/resources/TestData/Stocks-DE.csv create mode 100644 java/tsfile/src/test/resources/TestData/Stocks-UK.csv create mode 100644 java/tsfile/src/test/resources/TestData/Stocks-USA.csv create mode 100644 java/tsfile/src/test/resources/TestData/TY-Fuel.csv create mode 100644 java/tsfile/src/test/resources/TestData/TY-Transport.csv create mode 100644 java/tsfile/src/test/resources/TestData/Wind-Speed.csv diff --git a/cpp/third_party/zlib-1.3.1/treebuild.xml b/cpp/third_party/zlib-1.3.1/treebuild.xml index 930b00be4..8e030572a 100644 --- a/cpp/third_party/zlib-1.3.1/treebuild.xml +++ b/cpp/third_party/zlib-1.3.1/treebuild.xml @@ -1,103 +1,99 @@ - + - zip compression library - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + zip compression library + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + zip compression library + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -