From 91734b313ae3917c44a0dd516785f5ee423eac5b Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 29 Jul 2026 12:17:14 +0800 Subject: [PATCH 01/10] add property read-write interface --- .../tsfile/read/TsFileSequenceReader.java | 5 +++ .../read/v4/DeviceTableModelReader.java | 6 ++++ .../apache/tsfile/read/v4/ITsFileReader.java | 4 +++ .../v4/AbstractTableModelTsFileWriter.java | 6 ++++ .../apache/tsfile/write/v4/ITsFileWriter.java | 3 ++ .../tsfile/write/writer/TsFileIOWriter.java | 8 +++++ .../read/TsFileV4ReadWriteInterfacesTest.java | 33 +++++++++++++++++++ 7 files changed, 65 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 1fcabdab6..505b6aa8a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -607,6 +607,11 @@ public Map getTableSchemaMap() throws IOException { return getTableSchemaMap(null); } + /** Get the properties stored in the TsFile metadata. */ + public Map getTsFileProperties() throws IOException { + return readFileMetadata().getTsFileProperties(); + } + public Map getTableSchemaMap(LongConsumer ioSizeRecorder) throws IOException { if (tsFileMetaData != null && tsFileMetaData.hasTableSchemaMapCache()) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java index e7749d42b..4dffbbc01 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java @@ -78,6 +78,12 @@ public Optional getTableSchemas(String tableName) throws IOExceptio return Optional.ofNullable(tableSchemaMap.get(tableName.toLowerCase())); } + @Override + @TsFileApi + public Map getTsFileProperties() throws IOException { + return fileReader.getTsFileProperties(); + } + @TsFileApi public ResultSet query(String tableName, List columnNames, long startTime, long endTime) throws IOException, NoTableException, NoMeasurementException, ReadProcessException { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java index 995f73f64..0ade853c7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; public interface ITsFileReader extends AutoCloseable { @@ -48,6 +49,9 @@ ResultSet query( @TsFileApi List getAllTableSchema() throws IOException; + @TsFileApi + Map getTsFileProperties() throws IOException; + @TsFileApi void close(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index 94be1fdc1..2bb39ad57 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -259,6 +259,12 @@ protected Schema getSchema() { return fileWriter.getSchema(); } + @Override + @TsFileApi + public void addTsFileProperty(String key, String value) { + fileWriter.addTsFileProperty(key, value); + } + /** * calling this method to write the last data remaining in memory and close the normal and error * OutputStream. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java index 25ccb031b..15e5db30b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java @@ -36,4 +36,7 @@ public interface ITsFileWriter extends AutoCloseable { @TsFileApi void write(TSRecord record) throws IOException, WriteProcessException; + + @TsFileApi + void addTsFileProperty(String key, String value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index df6d99b20..a379e4314 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -143,6 +143,8 @@ public class TsFileIOWriter implements AutoCloseable { protected Map tableSizeMap = new HashMap<>(); + private final Map tsFileProperties = new HashMap<>(); + /** empty construct function. */ protected TsFileIOWriter() { setEncryptParam( @@ -246,6 +248,11 @@ public void setEncryptParam(EncryptParameter param) { } } + /** Add a custom property to the TsFile metadata. */ + public void addTsFileProperty(String key, String value) { + tsFileProperties.put(key, value); + } + public void addFlushListener(FlushChunkMetadataListener listener) { flushListeners.add(listener); } @@ -590,6 +597,7 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { tsFileMetadata.setTableSchemaMap(schema.getTableSchemaMap()); tsFileMetadata.setMetaOffset(metaOffset); tsFileMetadata.setBloomFilter(filter); + tsFileProperties.forEach(tsFileMetadata::addProperty); tsFileMetadata.addProperty("encryptLevel", encryptLevel); tsFileMetadata.addProperty("encryptType", encryptType); tsFileMetadata.addProperty("encryptKey", encryptKey); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java index 51aa64ce6..c8b0b5c0d 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java @@ -27,6 +27,7 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.v4.DeviceTableModelReader; +import org.apache.tsfile.read.v4.ITsFileReader; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsFileGeneratorForTest; import org.apache.tsfile.utils.TsFileGeneratorUtils; @@ -49,6 +50,38 @@ public class TsFileV4ReadWriteInterfacesTest { + @Test + public void testTsFileProperties() throws IOException { + String filePath = TsFileGeneratorForTest.getTestTsFilePath("properties", 0, 0, 0); + File file = new File(filePath); + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList(new MeasurementSchema("s1", TSDataType.INT32)), + Arrays.asList(ColumnCategory.FIELD)); + + try { + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + writer.addTsFileProperty("creator", "TsFileV4ReadWriteInterfacesTest"); + writer.addTsFileProperty("version", "1"); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + Assert.assertEquals( + "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator")); + Assert.assertEquals("1", reader.getTsFileProperties().get("version")); + } + try (ITsFileReader reader = new DeviceTableModelReader(file)) { + Assert.assertEquals( + "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator")); + Assert.assertEquals("1", reader.getTsFileProperties().get("version")); + } + } finally { + Files.deleteIfExists(file.toPath()); + } + } + @Test public void testWriteSomeColumns() throws IOException, WriteProcessException { String filePath = TsFileGeneratorForTest.getTestTsFilePath("db", 0, 0, 0); From 395e57a19c3a6c71be055ced50d6ebbec5d16aa5 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 29 Jul 2026 17:46:57 +0800 Subject: [PATCH 02/10] May record table point count --- .../tsfile/write/writer/TsFileIOWriter.java | 32 ++++++++++ .../tsfile/write/TsFileWriteApiTest.java | 61 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index a379e4314..79034b28e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -126,6 +126,7 @@ public class TsFileIOWriter implements AutoCloseable { protected LinkedList endPosInCMTForDevice = new LinkedList<>(); private volatile int chunkMetadataCount = 0; public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta"; + public static final String TABLE_POINT_COUNT_PROPERTY_PREFIX = "tablePointCount."; private boolean generateTableSchema = false; @@ -144,6 +145,8 @@ public class TsFileIOWriter implements AutoCloseable { protected Map tableSizeMap = new HashMap<>(); private final Map tsFileProperties = new HashMap<>(); + private final Map tablePointCountMap = new HashMap<>(); + private boolean recordTablePointCount; /** empty construct function. */ protected TsFileIOWriter() { @@ -165,6 +168,15 @@ public TsFileIOWriter(File file) throws IOException { this(file, TS_FILE_CONFIG); } + /** + * Creates a writer and optionally records the number of non-null field points for each table in + * the TsFile properties. + */ + public TsFileIOWriter(File file, boolean recordTablePointCount) throws IOException { + this(file); + this.recordTablePointCount = recordTablePointCount; + } + public TsFileIOWriter(File file, EncryptParameter param) throws IOException { this(file, TS_FILE_CONFIG, param); } @@ -368,6 +380,7 @@ public void writeChunk(Chunk chunk, ChunkMetadata chunkMetadata) throws IOExcept chunkHeader.getCompressionType(), out.getPosition(), chunkMetadata.getStatistics()); + currentChunkMetadata.setMask(chunkMetadata.getMask()); chunkHeader.serializeTo(out.wrapAsStream()); out.write(chunk.getData()); endCurrentChunk(); @@ -419,6 +432,10 @@ public void writeChunk(Chunk chunk) throws IOException { chunkHeader.getCompressionType(), out.getPosition(), chunk.getChunkStatistic()); + currentChunkMetadata.setMask( + (byte) + (chunkHeader.getChunkType() + & (TsFileConstant.TIME_COLUMN_MASK | TsFileConstant.VALUE_COLUMN_MASK))); chunkHeader.serializeTo(out.wrapAsStream()); out.write(chunk.getData()); endCurrentChunk(); @@ -428,6 +445,15 @@ public void writeChunk(Chunk chunk) throws IOException { public void endCurrentChunk() { this.currentChunkMetadataSize += currentChunkMetadata.getRetainedSizeInBytes(); chunkMetadataCount++; + if (recordTablePointCount + && currentChunkGroupDeviceId != null + && currentChunkGroupDeviceId.isTableModel() + && (currentChunkMetadata.getMask() & TsFileConstant.TIME_COLUMN_MASK) == 0) { + tablePointCountMap.merge( + currentChunkGroupDeviceId.getTableName(), + currentChunkMetadata.getNumOfPoints(), + Long::sum); + } chunkMetadataList.add(currentChunkMetadata); currentChunkMetadata = null; } @@ -598,6 +624,12 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { tsFileMetadata.setMetaOffset(metaOffset); tsFileMetadata.setBloomFilter(filter); tsFileProperties.forEach(tsFileMetadata::addProperty); + if (recordTablePointCount) { + tablePointCountMap.forEach( + (tableName, pointCount) -> + tsFileMetadata.addProperty( + TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, Long.toString(pointCount))); + } tsFileMetadata.addProperty("encryptLevel", encryptLevel); tsFileMetadata.addProperty("encryptType", encryptType); tsFileMetadata.addProperty("encryptKey", encryptKey); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index fcd761d3f..c8592afe1 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -1237,6 +1237,67 @@ public void queryTimestampStringBlobColumnsInTableModel() } } + /** + * Verifies that table point counting records only non-null FIELD values in TsFile properties. + * Table1 contains three s1 values and two non-null s2 values, while table2 contains two s1 + * values. Timestamps, TAG values, and the null s2 value are excluded, so the expected counts are + * five and two respectively. + */ + @Test + public void recordTablePointCountInProperties() throws IOException, WriteProcessException { + TableSchema tableSchema1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD), + new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD))); + TableSchema tableSchema2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); + Tablet tablet1 = + new Tablet( + "table1", + IMeasurementSchema.getMeasurementNameList(tableSchema1.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema1.getColumnSchemas()), + tableSchema1.getColumnTypes()); + for (int row = 0; row < 3; row++) { + tablet1.addTimestamp(row, row); + tablet1.addValue("device", row, "d1"); + tablet1.addValue("s1", row, row); + tablet1.addValue("s2", row, row == 2 ? null : row); + } + Tablet tablet2 = + new Tablet( + "table2", + IMeasurementSchema.getMeasurementNameList(tableSchema2.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema2.getColumnSchemas()), + tableSchema2.getColumnTypes()); + for (int row = 0; row < 2; row++) { + tablet2.addTimestamp(row, row); + tablet2.addValue("device", row, "d1"); + tablet2.addValue("s1", row, row); + } + + try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(f, true))) { + writer.registerTableSchema(tableSchema1); + writer.registerTableSchema(tableSchema2); + writer.writeTable(tablet1); + writer.writeTable(tablet2); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getAbsolutePath())) { + Map properties = reader.getTsFileProperties(); + Assert.assertEquals( + "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); + Assert.assertEquals( + "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); + } + } + @Test public void calculateTableSize() throws IOException, WriteProcessException { TableSchema tableSchema1 = From 95247656f01db2e165fe2d2fb07e06ccd29344cb Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 29 Jul 2026 18:40:31 +0800 Subject: [PATCH 03/10] Add perf test --- .../write/TablePointCountPerformanceTest.java | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java new file mode 100644 index 000000000..e31c4fd44 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.write; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class TablePointCountPerformanceTest { + + private static final String MODE_PROPERTY = "tablePointCountEnabled"; + private static final int TABLE1_ROW_COUNT = 200_000; + private static final int TABLE2_ROW_COUNT = 100_000; + private static final int BATCH_SIZE = 100; + private static final long TABLE1_POINT_COUNT = 333_333; + private static final long TABLE2_POINT_COUNT = 100_000; + + /** + * Measures one table point-count mode per JVM. Invoke this test twice with {@code + * -DtablePointCountEnabled=false} and {@code true}; each JVM performs its own warm-up and writes + * a mode-specific metrics file. Once both files exist, the test generates a comparison report + * containing the two JVM process IDs, file sizes, and measured write durations. + */ + @Test + public void measureTablePointCountInDedicatedJvm() throws IOException, WriteProcessException { + String configuredMode = System.getProperty(MODE_PROPERTY); + Assume.assumeTrue( + "Performance test requires -D" + MODE_PROPERTY + "=true|false", configuredMode != null); + Assert.assertTrue( + "Invalid -D" + MODE_PROPERTY + " value: " + configuredMode, + "true".equals(configuredMode) || "false".equals(configuredMode)); + boolean enabled = Boolean.parseBoolean(configuredMode); + + TableSchema tableSchema1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD), + new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD))); + TableSchema tableSchema2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); + List table1Tablets = createTablets(tableSchema1, TABLE1_ROW_COUNT, true); + List table2Tablets = createTablets(tableSchema2, TABLE2_ROW_COUNT, false); + + File targetDirectory = new File("target"); + File warmUpFile = new File(targetDirectory, "table-point-count-warm-up-" + enabled + ".tsfile"); + File measuredFile = new File(targetDirectory, "table-point-count-" + enabled + ".tsfile"); + try { + writeFile(warmUpFile, enabled, tableSchema1, tableSchema2, table1Tablets, table2Tablets); + Assert.assertTrue(warmUpFile.delete()); + + long startNanos = System.nanoTime(); + writeFile(measuredFile, enabled, tableSchema1, tableSchema2, table1Tablets, table2Tablets); + long elapsedNanos = System.nanoTime() - startNanos; + Assert.assertTrue(elapsedNanos > 0); + + verifyProperties(measuredFile, enabled); + long pid = ProcessHandle.current().pid(); + writeMetrics(targetDirectory, enabled, pid, measuredFile.length(), elapsedNanos); + generateReportIfBothModesExist(targetDirectory); + System.out.printf( + "Table point count enabled=%s: pid=%d, size=%d bytes, write=%d ns (%.3f ms)%n", + enabled, pid, measuredFile.length(), elapsedNanos, elapsedNanos / 1_000_000.0); + } finally { + warmUpFile.delete(); + measuredFile.delete(); + } + } + + private void writeFile( + File file, + boolean enabled, + TableSchema tableSchema1, + TableSchema tableSchema2, + List table1Tablets, + List table2Tablets) + throws IOException, WriteProcessException { + try (TsFileWriter writer = new TsFileWriter(new TsFileIOWriter(file, enabled))) { + writer.registerTableSchema(tableSchema1); + writer.registerTableSchema(tableSchema2); + for (Tablet tablet : table1Tablets) { + writer.writeTable(tablet); + } + for (Tablet tablet : table2Tablets) { + writer.writeTable(tablet); + } + } + } + + private List createTablets( + TableSchema tableSchema, int rowCount, boolean nullableSecondField) { + List columnNames = + IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas()); + List dataTypes = IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas()); + List tablets = new ArrayList<>((rowCount + BATCH_SIZE - 1) / BATCH_SIZE); + for (int startRow = 0; startRow < rowCount; startRow += BATCH_SIZE) { + int currentBatchSize = Math.min(BATCH_SIZE, rowCount - startRow); + Tablet tablet = + new Tablet( + tableSchema.getTableName(), + columnNames, + dataTypes, + tableSchema.getColumnTypes(), + currentBatchSize); + for (int row = 0; row < currentBatchSize; row++) { + int globalRow = startRow + row; + tablet.addTimestamp(row, globalRow); + tablet.addValue("device", row, "d1"); + tablet.addValue("s1", row, globalRow); + if (nullableSecondField) { + tablet.addValue("s2", row, globalRow % 3 == 0 ? null : globalRow); + } + } + tablets.add(tablet); + } + return tablets; + } + + private void verifyProperties(File file, boolean enabled) throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + Map properties = reader.getTsFileProperties(); + String table1Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1"; + String table2Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2"; + if (enabled) { + Assert.assertEquals(Long.toString(TABLE1_POINT_COUNT), properties.get(table1Key)); + Assert.assertEquals(Long.toString(TABLE2_POINT_COUNT), properties.get(table2Key)); + } else { + Assert.assertFalse(properties.containsKey(table1Key)); + Assert.assertFalse(properties.containsKey(table2Key)); + } + } + } + + private void writeMetrics( + File targetDirectory, boolean enabled, long pid, long fileSize, long elapsedNanos) + throws IOException { + Properties metrics = new Properties(); + metrics.setProperty("enabled", Boolean.toString(enabled)); + metrics.setProperty("pid", Long.toString(pid)); + metrics.setProperty("fileSize", Long.toString(fileSize)); + metrics.setProperty("elapsedNanos", Long.toString(elapsedNanos)); + metrics.setProperty("rows", Integer.toString(TABLE1_ROW_COUNT + TABLE2_ROW_COUNT)); + metrics.setProperty("points", Long.toString(TABLE1_POINT_COUNT + TABLE2_POINT_COUNT)); + try (FileOutputStream output = new FileOutputStream(metricsFile(targetDirectory, enabled))) { + metrics.store(output, "Table point count performance metrics"); + } + } + + private void generateReportIfBothModesExist(File targetDirectory) throws IOException { + File disabledMetricsFile = metricsFile(targetDirectory, false); + File enabledMetricsFile = metricsFile(targetDirectory, true); + if (!disabledMetricsFile.isFile() || !enabledMetricsFile.isFile()) { + return; + } + Properties disabled = loadMetrics(disabledMetricsFile); + Properties enabled = loadMetrics(enabledMetricsFile); + long disabledPid = Long.parseLong(disabled.getProperty("pid")); + long enabledPid = Long.parseLong(enabled.getProperty("pid")); + Assert.assertNotEquals("Modes must run in separate JVMs", disabledPid, enabledPid); + long disabledSize = Long.parseLong(disabled.getProperty("fileSize")); + long enabledSize = Long.parseLong(enabled.getProperty("fileSize")); + long disabledNanos = Long.parseLong(disabled.getProperty("elapsedNanos")); + long enabledNanos = Long.parseLong(enabled.getProperty("elapsedNanos")); + Assert.assertTrue(enabledSize > disabledSize); + + String report = + String.format( + "# Table Point Count Performance Report%n%n" + + "## Dataset%n%n" + + "- table1: %,d rows, %,d non-null FIELD points%n" + + "- table2: %,d rows, %,d non-null FIELD points%n" + + "- Batch size: %,d rows%n" + + "- Total: %,d rows, %,d non-null FIELD points%n%n" + + "## Results%n%n" + + "| Point counting | JVM PID | File size (bytes) | Write time (ns) | Write time (ms) |%n" + + "|---|---:|---:|---:|---:|%n" + + "| Disabled | %d | %,d | %,d | %.3f |%n" + + "| Enabled | %d | %,d | %,d | %.3f |%n%n" + + "- File-size delta: %,d bytes%n" + + "- File-size overhead: %.3f%%%n" + + "- Write-time delta: %,d ns (%.3f ms)%n" + + "- Write-time overhead: %.3f%%%n%n" + + "> Each mode ran in a separate JVM and performed an independent warm-up. Write " + + "time includes writer construction, schema registration, batched writes, " + + "metadata serialization, force, and close. Timing is diagnostic and is not used " + + "as a CI pass/fail threshold.%n", + TABLE1_ROW_COUNT, + TABLE1_POINT_COUNT, + TABLE2_ROW_COUNT, + TABLE2_POINT_COUNT, + BATCH_SIZE, + TABLE1_ROW_COUNT + TABLE2_ROW_COUNT, + TABLE1_POINT_COUNT + TABLE2_POINT_COUNT, + disabledPid, + disabledSize, + disabledNanos, + disabledNanos / 1_000_000.0, + enabledPid, + enabledSize, + enabledNanos, + enabledNanos / 1_000_000.0, + enabledSize - disabledSize, + (enabledSize - disabledSize) * 100.0 / disabledSize, + enabledNanos - disabledNanos, + (enabledNanos - disabledNanos) / 1_000_000.0, + (enabledNanos - disabledNanos) * 100.0 / disabledNanos); + File reportFile = new File(targetDirectory, "table-point-count-performance-report.md"); + Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8)); + System.out.printf("Table point-count report: %s%n", reportFile.getAbsolutePath()); + } + + private Properties loadMetrics(File file) throws IOException { + Properties metrics = new Properties(); + try (FileInputStream input = new FileInputStream(file)) { + metrics.load(input); + } + return metrics; + } + + private File metricsFile(File targetDirectory, boolean enabled) { + return new File(targetDirectory, "table-point-count-" + enabled + ".properties"); + } +} From 5ffa041a12f9ee750b7723a17530f55f5ea52ed9 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 29 Jul 2026 18:59:45 +0800 Subject: [PATCH 04/10] Add table point count repair tool --- .../apache/tsfile/i18n/messages.properties | 7 + .../apache/tsfile/i18n/messages_zh.properties | 7 + .../utils/TsFileTablePointCountTool.java | 243 ++++++++++++++++++ .../utils/TsFileTablePointCountToolTest.java | 116 +++++++++ 4 files changed, 373 insertions(+) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties index 98909f7a6..9f6f35857 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties @@ -1548,3 +1548,10 @@ log.tools.csv_read_error = Error reading CSV file: %1$s # CsvSourceReader — error closing CSV reader log.tools.csv_close_reader_error = Error closing CSV reader + +# TsFileTablePointCountTool +error.utils.table_point_count_tool_usage = Usage: TsFileTablePointCountTool +error.utils.table_point_count_tool_file_not_found = TsFile does not exist: %1$s +error.utils.table_point_count_tool_incomplete_file = TsFile is incomplete: %1$s +error.utils.table_point_count_tool_copy_failed = Failed to copy the TsFile data and metadata prefix +info.utils.table_point_count_tool_result = Processed %1$s: %2$s diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties index a1a437cfb..17c341af2 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties @@ -1548,3 +1548,10 @@ log.tools.csv_read_error = 读取 CSV 文件出错: %1$s # CsvSourceReader — error closing CSV reader log.tools.csv_close_reader_error = 关闭 CSV reader 出错 + +# TsFileTablePointCountTool +error.utils.table_point_count_tool_usage = 用法：TsFileTablePointCountTool +error.utils.table_point_count_tool_file_not_found = TsFile 不存在：%1$s +error.utils.table_point_count_tool_incomplete_file = TsFile 不完整：%1$s +error.utils.table_point_count_tool_copy_failed = 复制 TsFile 数据和元数据前缀失败 +info.utils.table_point_count_tool_result = 已处理 %1$s：%2$s diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java new file mode 100644 index 000000000..923b7c31d --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.utils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Detects and backfills table-level point-count properties in a complete TsFile. */ +public final class TsFileTablePointCountTool { + + private TsFileTablePointCountTool() {} + + public enum UpdateStatus { + UPDATED, + ALREADY_PRESENT, + NO_TABLE + } + + public static void main(String[] args) throws IOException { + if (args.length != 1) { + System.err.println(Messages.get("error.utils.table_point_count_tool_usage")); + return; + } + File file = new File(args[0]); + UpdateStatus status = updateTablePointCountIfMissing(file); + System.out.println( + Messages.format( + "info.utils.table_point_count_tool_result", file.getAbsolutePath(), status)); + } + + /** + * Returns whether every table in the file has a valid non-negative point-count property. + * Tree-model-only files return {@code false}. + */ + public static boolean containsTablePointCount(File file) throws IOException { + try (TsFileSequenceReader reader = openCompleteFile(file)) { + Map tableSchemas = reader.getTableSchemaMap(); + return !tableSchemas.isEmpty() + && hasAllTablePointCountProperties(reader.getTsFileProperties(), tableSchemas.keySet()); + } + } + + /** + * Adds table point-count properties when they are missing. Existing complete properties are left + * untouched. The source file is replaced only after a complete rewritten copy has been forced to + * disk. + */ + public static UpdateStatus updateTablePointCountIfMissing(File file) throws IOException { + TsFileMetadata metadata; + long fileMetadataPosition; + Map tablePointCounts; + try (TsFileSequenceReader reader = openCompleteFile(file)) { + reader.setEnableCacheTableSchemaMap(); + metadata = reader.readFileMetadata(); + Map tableSchemas = metadata.getTableSchemaMap(); + if (tableSchemas.isEmpty()) { + return UpdateStatus.NO_TABLE; + } + if (hasAllTablePointCountProperties(metadata.getTsFileProperties(), tableSchemas.keySet())) { + return UpdateStatus.ALREADY_PRESENT; + } + tablePointCounts = scanTablePointCounts(reader, tableSchemas); + fileMetadataPosition = reader.getFileMetadataPos(); + } + + tablePointCounts.forEach( + (tableName, pointCount) -> + metadata.addProperty( + TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, + Long.toString(pointCount))); + rewriteFileMetadata(file.toPath(), fileMetadataPosition, metadata); + return UpdateStatus.UPDATED; + } + + static Map scanTablePointCounts( + TsFileSequenceReader reader, Map tableSchemas) throws IOException { + Map> fieldNamesByTable = new HashMap<>(); + Map tablePointCounts = new HashMap<>(); + for (Map.Entry tableEntry : tableSchemas.entrySet()) { + List columns = tableEntry.getValue().getColumnSchemas(); + List columnCategories = tableEntry.getValue().getColumnTypes(); + Set fieldNames = new HashSet<>(); + for (int i = 0; i < columns.size(); i++) { + if (columnCategories.get(i) == ColumnCategory.FIELD) { + fieldNames.add(columns.get(i).getMeasurementName()); + } + } + fieldNamesByTable.put(tableEntry.getKey(), fieldNames); + tablePointCounts.put(tableEntry.getKey(), 0L); + } + + for (IDeviceID device : reader.getAllDevices()) { + String tableName = device.getTableName(); + Set fieldNames = fieldNamesByTable.getOrDefault(tableName, Collections.emptySet()); + if (fieldNames.isEmpty()) { + continue; + } + Map> chunkMetadataByMeasurement = + reader.readChunkMetadataInDevice(device); + for (String fieldName : fieldNames) { + for (ChunkMetadata chunkMetadata : + chunkMetadataByMeasurement.getOrDefault(fieldName, Collections.emptyList())) { + tablePointCounts.merge( + tableName, (long) chunkMetadata.getStatistics().getCount(), Long::sum); + } + } + } + return tablePointCounts; + } + + private static TsFileSequenceReader openCompleteFile(File file) throws IOException { + if (!file.isFile()) { + throw new IOException( + Messages.format("error.utils.table_point_count_tool_file_not_found", file)); + } + TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath()); + try { + if (!reader.isComplete()) { + throw new IOException( + Messages.format("error.utils.table_point_count_tool_incomplete_file", file)); + } + return reader; + } catch (IOException | RuntimeException e) { + try { + reader.close(); + } catch (IOException closeException) { + e.addSuppressed(closeException); + } + throw e; + } + } + + private static boolean hasAllTablePointCountProperties( + Map properties, Set tableNames) { + if (properties == null) { + return false; + } + for (String tableName : tableNames) { + String value = properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName); + try { + if (value == null || Long.parseLong(value) < 0) { + return false; + } + } catch (NumberFormatException e) { + return false; + } + } + return true; + } + + private static void rewriteFileMetadata( + Path sourceFile, long fileMetadataPosition, TsFileMetadata metadata) throws IOException { + Path absoluteSource = sourceFile.toAbsolutePath(); + Path temporaryFile = + Files.createTempFile( + absoluteSource.getParent(), + absoluteSource.getFileName().toString(), + ".point-count.tmp"); + try { + try (FileChannel sourceChannel = FileChannel.open(absoluteSource, StandardOpenOption.READ); + FileChannel targetChannel = + FileChannel.open( + temporaryFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + copyPrefix(sourceChannel, targetChannel, fileMetadataPosition); + targetChannel.position(fileMetadataPosition); + OutputStream output = Channels.newOutputStream(targetChannel); + int metadataSize = metadata.serializeTo(output); + ReadWriteIOUtils.write(metadataSize, output); + output.write(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET)); + output.flush(); + targetChannel.force(true); + } + replaceSourceFile(temporaryFile, absoluteSource); + } finally { + Files.deleteIfExists(temporaryFile); + } + } + + private static void copyPrefix( + FileChannel sourceChannel, FileChannel targetChannel, long prefixLength) throws IOException { + long copied = 0; + while (copied < prefixLength) { + long currentCopied = sourceChannel.transferTo(copied, prefixLength - copied, targetChannel); + if (currentCopied <= 0) { + throw new IOException(Messages.get("error.utils.table_point_count_tool_copy_failed")); + } + copied += currentCopied; + } + } + + private static void replaceSourceFile(Path temporaryFile, Path sourceFile) throws IOException { + try { + Files.move( + temporaryFile, + sourceFile, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporaryFile, sourceFile, StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java new file mode 100644 index 000000000..edf12081d --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.utils; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Map; + +public class TsFileTablePointCountToolTest { + + /** + * Creates a two-table file without point-count properties. The repair tool must count only + * non-null FIELD values from chunk statistics, preserve unrelated properties and file data, and + * leave the file byte-for-byte unchanged when run again. + */ + @Test + public void backfillMissingTablePointCountProperties() throws Exception { + File file = File.createTempFile("table-point-count-tool", ".tsfile"); + try { + TableSchema tableSchema1 = + new TableSchema( + "table1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD), + new ColumnSchema("s2", TSDataType.INT32, ColumnCategory.FIELD))); + TableSchema tableSchema2 = + new TableSchema( + "table2", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); + try (TsFileIOWriter ioWriter = new TsFileIOWriter(file); + TsFileWriter writer = new TsFileWriter(ioWriter)) { + ioWriter.addTsFileProperty("custom.property", "preserved"); + writer.registerTableSchema(tableSchema1); + writer.registerTableSchema(tableSchema2); + writer.writeTable(createTablet(tableSchema1, 3, true)); + writer.writeTable(createTablet(tableSchema2, 2, false)); + } + + Assert.assertFalse(TsFileTablePointCountTool.containsTablePointCount(file)); + Assert.assertEquals( + TsFileTablePointCountTool.UpdateStatus.UPDATED, + TsFileTablePointCountTool.updateTablePointCountIfMissing(file)); + Assert.assertTrue(TsFileTablePointCountTool.containsTablePointCount(file)); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + Assert.assertTrue(reader.isComplete()); + Map properties = reader.getTsFileProperties(); + Assert.assertEquals("preserved", properties.get("custom.property")); + Assert.assertEquals( + "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); + Assert.assertEquals( + "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); + Assert.assertEquals(2, reader.getAllDevices().size()); + } + + byte[] repairedFile = Files.readAllBytes(file.toPath()); + Assert.assertEquals( + TsFileTablePointCountTool.UpdateStatus.ALREADY_PRESENT, + TsFileTablePointCountTool.updateTablePointCountIfMissing(file)); + Assert.assertArrayEquals(repairedFile, Files.readAllBytes(file.toPath())); + } finally { + Files.deleteIfExists(file.toPath()); + } + } + + private Tablet createTablet(TableSchema tableSchema, int rowCount, boolean nullableSecondField) { + Tablet tablet = + new Tablet( + tableSchema.getTableName(), + IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas()), + tableSchema.getColumnTypes()); + for (int row = 0; row < rowCount; row++) { + tablet.addTimestamp(row, row); + tablet.addValue("device", row, "d1"); + tablet.addValue("s1", row, row); + if (nullableSecondField) { + tablet.addValue("s2", row, row == rowCount - 1 ? null : row); + } + } + return tablet; + } +} From efed191dd0ef4f33efa784a6cf2ba6c77ab16709 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 30 Jul 2026 12:32:11 +0800 Subject: [PATCH 05/10] add point-count tool --- .../utils/TsFileTablePointCountTool.java | 115 +++++++++++++++--- .../utils/TsFileTablePointCountToolTest.java | 27 +++- 2 files changed, 119 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java index 923b7c31d..d0d73d59a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java @@ -20,9 +20,9 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.TsFileMetadata; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TsFileSequenceReader; @@ -42,9 +42,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; /** Detects and backfills table-level point-count properties in a complete TsFile. */ public final class TsFileTablePointCountTool { @@ -130,19 +133,20 @@ static Map scanTablePointCounts( tablePointCounts.put(tableEntry.getKey(), 0L); } - for (IDeviceID device : reader.getAllDevices()) { + Iterator>> metadataIterator = + reader.iterAllTimeseriesMetadata(false, false); + while (metadataIterator.hasNext()) { + Pair> deviceMetadata = metadataIterator.next(); + IDeviceID device = deviceMetadata.left; String tableName = device.getTableName(); Set fieldNames = fieldNamesByTable.getOrDefault(tableName, Collections.emptySet()); if (fieldNames.isEmpty()) { continue; } - Map> chunkMetadataByMeasurement = - reader.readChunkMetadataInDevice(device); - for (String fieldName : fieldNames) { - for (ChunkMetadata chunkMetadata : - chunkMetadataByMeasurement.getOrDefault(fieldName, Collections.emptyList())) { + for (TimeseriesMetadata timeseriesMetadata : deviceMetadata.right) { + if (fieldNames.contains(timeseriesMetadata.getMeasurementId())) { tablePointCounts.merge( - tableName, (long) chunkMetadata.getStatistics().getCount(), Long::sum); + tableName, (long) timeseriesMetadata.getStatistics().getCount(), Long::sum); } } } @@ -192,24 +196,34 @@ private static boolean hasAllTablePointCountProperties( private static void rewriteFileMetadata( Path sourceFile, long fileMetadataPosition, TsFileMetadata metadata) throws IOException { Path absoluteSource = sourceFile.toAbsolutePath(); + // Keep the temporary file beside the source so reflink and atomic move can stay on the same + // file system. The source is not replaced until the rewritten temporary file has been forced. Path temporaryFile = Files.createTempFile( absoluteSource.getParent(), absoluteSource.getFileName().toString(), ".point-count.tmp"); try { - try (FileChannel sourceChannel = FileChannel.open(absoluteSource, StandardOpenOption.READ); - FileChannel targetChannel = - FileChannel.open( - temporaryFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { - copyPrefix(sourceChannel, targetChannel, fileMetadataPosition); - targetChannel.position(fileMetadataPosition); - OutputStream output = Channels.newOutputStream(targetChannel); - int metadataSize = metadata.serializeTo(output); - ReadWriteIOUtils.write(metadataSize, output); - output.write(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET)); - output.flush(); - targetChannel.force(true); + if (tryCreateReflink(absoluteSource, temporaryFile)) { + // A reflink shares unchanged data blocks with the source. Truncating and rewriting only the + // TsFileMetadata tail therefore triggers copy-on-write for the affected tail blocks. + try (FileChannel targetChannel = + FileChannel.open(temporaryFile, StandardOpenOption.WRITE)) { + targetChannel.truncate(fileMetadataPosition); + writeFileMetadata(targetChannel, fileMetadataPosition, metadata); + } + } else { + // Reflink is an optional optimization. Copy the immutable data/index prefix when the + // command or file system does not support copy-on-write cloning. + try (FileChannel sourceChannel = FileChannel.open(absoluteSource, StandardOpenOption.READ); + FileChannel targetChannel = + FileChannel.open( + temporaryFile, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + copyPrefix(sourceChannel, targetChannel, fileMetadataPosition); + writeFileMetadata(targetChannel, fileMetadataPosition, metadata); + } } replaceSourceFile(temporaryFile, absoluteSource); } finally { @@ -217,6 +231,65 @@ private static void rewriteFileMetadata( } } + /** + * Attempts a same-file-system copy-on-write clone using the native platform command. GNU/Linux + * uses {@code cp --reflink=always}, while macOS uses {@code cp -c}. Unsupported platforms, + * missing commands, unsupported file systems, invalid clone results, and command failures return + * {@code false} so the caller can safely fall back to a full prefix copy. + */ + private static boolean tryCreateReflink(Path sourceFile, Path targetFile) throws IOException { + String operatingSystem = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + List command; + if (operatingSystem.contains("linux")) { + command = + List.of("cp", "--reflink=always", "--", sourceFile.toString(), targetFile.toString()); + } else if (operatingSystem.contains("mac")) { + command = List.of("cp", "-c", sourceFile.toString(), targetFile.toString()); + } else { + return false; + } + + try { + Process process = + new ProcessBuilder(command) + .redirectErrorStream(true) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .start(); + // A reflink should complete quickly regardless of file size. Bound the external command so + // an unavailable or unhealthy file system cannot block the repair indefinitely. + if (!process.waitFor(30, TimeUnit.SECONDS)) { + process.destroyForcibly(); + // Wait for termination before the fallback path opens and rewrites the same temporary file. + process.waitFor(); + return false; + } + // Do not trust the exit code alone: verify that the clone produced a complete file before + // truncating its metadata tail. + return process.exitValue() == 0 + && Files.isRegularFile(targetFile) + && Files.size(targetFile) == Files.size(sourceFile); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } catch (IOException | SecurityException e) { + return false; + } + } + + /** Writes the replacement TsFileMetadata, its serialized size, and the trailing TsFile magic. */ + private static void writeFileMetadata( + FileChannel targetChannel, long fileMetadataPosition, TsFileMetadata metadata) + throws IOException { + targetChannel.position(fileMetadataPosition); + OutputStream output = Channels.newOutputStream(targetChannel); + int metadataSize = metadata.serializeTo(output); + ReadWriteIOUtils.write(metadataSize, output); + output.write(TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET)); + output.flush(); + targetChannel.force(true); + } + + /** Copies the immutable data and metadata-index prefix used by the non-reflink fallback. */ private static void copyPrefix( FileChannel sourceChannel, FileChannel targetChannel, long prefixLength) throws IOException { long copied = 0; @@ -231,12 +304,14 @@ private static void copyPrefix( private static void replaceSourceFile(Path temporaryFile, Path sourceFile) throws IOException { try { + // Prefer an atomic replacement so readers never observe a partially rewritten TsFile. Files.move( temporaryFile, sourceFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (AtomicMoveNotSupportedException e) { + // Some file systems do not provide atomic move even within one directory. Files.move(temporaryFile, sourceFile, StandardCopyOption.REPLACE_EXISTING); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java index edf12081d..a1b604728 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java @@ -20,7 +20,9 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.write.TsFileWriter; @@ -34,14 +36,15 @@ import java.io.File; import java.nio.file.Files; import java.util.Arrays; +import java.util.List; import java.util.Map; public class TsFileTablePointCountToolTest { /** - * Creates a two-table file without point-count properties. The repair tool must count only - * non-null FIELD values from chunk statistics, preserve unrelated properties and file data, and - * leave the file byte-for-byte unchanged when run again. + * Creates a two-table file without point-count properties. The repair tool must traverse the + * metadata index tree and count only non-null FIELD values, preserve unrelated properties and + * file data, and leave the file byte-for-byte unchanged when run again. */ @Test public void backfillMissingTablePointCountProperties() throws Exception { @@ -69,6 +72,24 @@ public void backfillMissingTablePointCountProperties() throws Exception { writer.writeTable(createTablet(tableSchema2, 2, false)); } + try (TsFileSequenceReader reader = + new TsFileSequenceReader(file.getAbsolutePath()) { + @Override + public List getAllDevices() { + throw new AssertionError("The tool must not enumerate all devices"); + } + + @Override + public Map> readChunkMetadataInDevice(IDeviceID device) { + throw new AssertionError("The tool must not query chunk metadata by device"); + } + }) { + Map pointCounts = + TsFileTablePointCountTool.scanTablePointCounts(reader, reader.getTableSchemaMap()); + Assert.assertEquals(Long.valueOf(5), pointCounts.get("table1")); + Assert.assertEquals(Long.valueOf(2), pointCounts.get("table2")); + } + Assert.assertFalse(TsFileTablePointCountTool.containsTablePointCount(file)); Assert.assertEquals( TsFileTablePointCountTool.UpdateStatus.UPDATED, From 9d097aa7362cc2f798fd8b3ac5462784f410e088 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 30 Jul 2026 15:42:31 +0800 Subject: [PATCH 06/10] add point-count tool --- java/tools/README-zh.md | 19 +++++++ java/tools/README.md | 24 +++++++++ .../tools/tsfile-table-point-count.bat | 49 ++++++++++++++++++ .../tools/tsfile-table-point-count.sh | 50 +++++++++++++++++++ java/tools/src/assembly/tools.xml | 9 ++++ 5 files changed, 151 insertions(+) create mode 100644 java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat create mode 100644 java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh diff --git a/java/tools/README-zh.md b/java/tools/README-zh.md index 46c615ed0..4c1669a8c 100644 --- a/java/tools/README-zh.md +++ b/java/tools/README-zh.md @@ -229,3 +229,22 @@ arrow2tsfile.bat --source .\data\arrow --target .\output --fail_dir .\failed - 多批次：`{源文件名}_1.tsfile`、`{源文件名}_2.tsfile`、... - 表名与输出文件名相互独立——表名来自 schema 或 `--table_name`，文件名来自源文件。 +## 表点数统计工具 + +`tools\tsfile-table-point-count.bat` 用于检查完整 TsFile 是否包含表级点数统计属性，并在属性缺失时补写。每张表的点数是其所有 FIELD 列中非空值的总数，不统计 TAG 列和时间列。 + +设置 `JAVA_HOME` 后，传入且仅传入一个 TsFile 路径： + +```bat +tools\tsfile-table-point-count.bat C:\data\example.tsfile +``` + +工具会输出以下状态之一： + +| 状态 | 说明 | +|------|------| +| `UPDATED` | 已计算缺失的点数统计属性并写回文件。 | +| `ALREADY_PRESENT` | 每张表都已有有效的点数统计属性，未修改文件。 | +| `NO_TABLE` | 文件不包含表 Schema，例如仅含树模型数据的 TsFile，未修改文件。 | + +仅当状态为 `UPDATED` 时，工具才会原地修改输入文件。工具先在源文件所在目录生成完整的临时文件，确保重写后的元数据落盘后再替换源文件。运行前请确保目录可写、磁盘空间足以存放临时副本，并备份重要文件。不存在的文件和未写完整的 TsFile 会被拒绝处理。 diff --git a/java/tools/README.md b/java/tools/README.md index ad1f40215..e6584fe2a 100644 --- a/java/tools/README.md +++ b/java/tools/README.md @@ -228,3 +228,27 @@ arrow2tsfile.bat --source .\data\arrow --target .\output --fail_dir .\failed - Multiple batches: `{source_basename}_1.tsfile`, `{source_basename}_2.tsfile`, ... - Table name and output filename are independent — table name comes from schema or `--table_name`, filename comes from source file. +## Table Point Count Tool + +`tools\tsfile-table-point-count.bat` checks whether a complete TsFile contains table-level +point-count properties and backfills them when they are missing. For each table, the point count is +the total number of non-null values in its FIELD columns; TAG and time columns are not counted. + +Set `JAVA_HOME`, then run the tool with exactly one TsFile path: + +```bat +tools\tsfile-table-point-count.bat C:\data\example.tsfile +``` + +The tool prints one of the following statuses: + +| Status | Description | +|--------|-------------| +| `UPDATED` | Missing point-count properties were calculated and written to the file. | +| `ALREADY_PRESENT` | Every table already had a valid point-count property; the file was not modified. | +| `NO_TABLE` | The file contains no table schema, for example a tree-model-only TsFile; the file was not modified. | + +The tool modifies the input file in place only for `UPDATED`. It writes a complete temporary copy +next to the source file and replaces the source after the rewritten metadata has been flushed. Make +sure the directory is writable, leave enough free space for a temporary copy, and back up important +files before running the tool. Missing files and incomplete TsFiles are rejected. diff --git a/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat new file mode 100644 index 000000000..a89d6c13d --- /dev/null +++ b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat @@ -0,0 +1,49 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +setlocal enabledelayedexpansion + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0.. +if NOT DEFINED TSFILE_HOME set TSFILE_HOME=%CD% +popd + +set JAVA_OPTS=-ea^ + -DTSFILE_HOME="%TSFILE_HOME%" + +if NOT DEFINED JAVA_HOME goto :err + +echo ------------------------------------------ +echo Starting TsFile Table Point Count Tool +echo ------------------------------------------ + +set CLASSPATH="%TSFILE_HOME%\lib\*" +set MAIN_CLASS=org.apache.tsfile.utils.TsFileTablePointCountTool + +@REM The argument is the TsFile to inspect. The tool rewrites its properties only when table-level +@REM point-count statistics are missing. +"%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %* +exit /b %ERRORLEVEL% + +:err +echo JAVA_HOME environment variable must be set! +set ret_code=1 +exit /b %ret_code% diff --git a/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh new file mode 100644 index 000000000..34c902628 --- /dev/null +++ b/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo ------------------------------------------ +echo Starting TsFile Table Point Count Tool +echo ------------------------------------------ + +if [ -z "${TSFILE_HOME}" ]; then + export TSFILE_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +if [ -z $JAVA ] ; then + echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr + exit 1; +fi + +CLASSPATH=${TSFILE_HOME}/lib/* +MAIN_CLASS=org.apache.tsfile.utils.TsFileTablePointCountTool + +# The argument is the TsFile to inspect. The tool rewrites its properties only when table-level +# point-count statistics are missing. +exec "$JAVA" -DTSFILE_HOME=${TSFILE_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" diff --git a/java/tools/src/assembly/tools.xml b/java/tools/src/assembly/tools.xml index 31e9c81a0..b58e8b8e2 100644 --- a/java/tools/src/assembly/tools.xml +++ b/java/tools/src/assembly/tools.xml @@ -69,5 +69,14 @@ ${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/arrow2tsfile.bat tools/arrow2tsfile.bat + + ${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/tsfile-table-point-count.sh + tools/tsfile-table-point-count.sh + 0755 + + + ${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/tsfile-table-point-count.bat + tools/tsfile-table-point-count.bat + From d327fae7a7d5fce0e0c6ab10e414a8c5ddcc2093 Mon Sep 17 00:00:00 2001 From: Jiang Tian Date: Thu, 30 Jul 2026 17:34:16 +0800 Subject: [PATCH 07/10] supplement usage for other os Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- java/tools/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/java/tools/README.md b/java/tools/README.md index e6584fe2a..18c5cd780 100644 --- a/java/tools/README.md +++ b/java/tools/README.md @@ -230,15 +230,17 @@ arrow2tsfile.bat --source .\data\arrow --target .\output --fail_dir .\failed ## Table Point Count Tool -`tools\tsfile-table-point-count.bat` checks whether a complete TsFile contains table-level +`tools/tsfile-table-point-count.sh` (Linux/macOS) and `tools\tsfile-table-point-count.bat` (Windows) check whether a complete TsFile contains table-level point-count properties and backfills them when they are missing. For each table, the point count is the total number of non-null values in its FIELD columns; TAG and time columns are not counted. Set `JAVA_HOME`, then run the tool with exactly one TsFile path: -```bat -tools\tsfile-table-point-count.bat C:\data\example.tsfile -``` + # Linux/macOS + tools/tsfile-table-point-count.sh /data/example.tsfile + + :: Windows + tools\tsfile-table-point-count.bat C:\data\example.tsfile The tool prints one of the following statuses: From 339a1f14a4170362b68f7582645e0df35673cfea Mon Sep 17 00:00:00 2001 From: Jiang Tian Date: Thu, 30 Jul 2026 17:34:28 +0800 Subject: [PATCH 08/10] supplement usage for other os Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- java/tools/README-zh.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/java/tools/README-zh.md b/java/tools/README-zh.md index 4c1669a8c..7ff5ef6e4 100644 --- a/java/tools/README-zh.md +++ b/java/tools/README-zh.md @@ -231,13 +231,15 @@ arrow2tsfile.bat --source .\data\arrow --target .\output --fail_dir .\failed ## 表点数统计工具 -`tools\tsfile-table-point-count.bat` 用于检查完整 TsFile 是否包含表级点数统计属性，并在属性缺失时补写。每张表的点数是其所有 FIELD 列中非空值的总数，不统计 TAG 列和时间列。 +`tools/tsfile-table-point-count.sh`（Linux/macOS）和 `tools\tsfile-table-point-count.bat`（Windows）用于检查完整 TsFile 是否包含表级点数统计属性，并在属性缺失时补写。每张表的点数是其所有 FIELD 列中非空值的总数，不统计 TAG 列和时间列。 设置 `JAVA_HOME` 后，传入且仅传入一个 TsFile 路径： -```bat -tools\tsfile-table-point-count.bat C:\data\example.tsfile -``` + # Linux/macOS + tools/tsfile-table-point-count.sh /data/example.tsfile + + :: Windows + tools\tsfile-table-point-count.bat C:\data\example.tsfile 工具会输出以下状态之一： From 888166d806c856748ca26535f281482caf89ba45 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 30 Jul 2026 18:26:02 +0800 Subject: [PATCH 09/10] use byte[] for properties --- .../tsfile/file/metadata/TsFileMetadata.java | 74 ++++++++++++------- .../tsfile/read/TsFileSequenceReader.java | 2 +- .../read/v4/DeviceTableModelReader.java | 2 +- .../apache/tsfile/read/v4/ITsFileReader.java | 2 +- .../utils/TsFileTablePointCountTool.java | 8 +- .../v4/AbstractTableModelTsFileWriter.java | 2 +- .../apache/tsfile/write/v4/ITsFileWriter.java | 2 +- .../tsfile/write/writer/TsFileIOWriter.java | 14 ++-- .../file/metadata/TsFileMetadataTest.java | 11 ++- .../tsfile/file/metadata/utils/Utils.java | 21 +++++- .../read/TsFileV4ReadWriteInterfacesTest.java | 22 ++++-- .../utils/TsFileTablePointCountToolTest.java | 18 +++-- .../write/TablePointCountPerformanceTest.java | 10 ++- .../tsfile/write/TsFileWriteApiTest.java | 12 +-- 14 files changed, 129 insertions(+), 71 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java index 318728553..7caae06b2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java @@ -19,6 +19,7 @@ package org.apache.tsfile.file.metadata; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.compatibility.DeserializeConfig; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.exception.encrypt.EncryptException; @@ -45,7 +46,7 @@ public class TsFileMetadata { private Map tableMetadataIndexNodeMap; private Map tableSchemaMap; private boolean hasTableSchemaMapCache; - private Map tsFileProperties; + private Map tsFileProperties; // offset of MetaMarker.SEPARATOR private long metaOffset; @@ -117,21 +118,27 @@ public static TsFileMetadata deserializeFrom( if (buffer.hasRemaining()) { int propertiesSize = ReadWriteForEncodingUtils.readVarInt(buffer); - Map propertiesMap = new HashMap<>(); + Map propertiesMap = new HashMap<>(); for (int i = 0; i < propertiesSize; i++) { String key = ReadWriteIOUtils.readVarIntString(buffer); - String value = ReadWriteIOUtils.readVarIntString(buffer); + int valueSize = ReadWriteForEncodingUtils.readVarInt(buffer); + byte[] value = null; + if (valueSize >= 0) { + value = new byte[valueSize]; + buffer.get(value); + } propertiesMap.put(key, value); } + String encryptLevel = getPropertyAsString(propertiesMap, "encryptLevel"); // if the file is not encrypted, set the default value(for compatible reason) - if (!propertiesMap.containsKey("encryptLevel") || propertiesMap.get("encryptLevel") == null) { - propertiesMap.put("encryptLevel", "0"); - propertiesMap.put("encryptType", "org.apache.tsfile.encrypt.UNENCRYPTED"); - propertiesMap.put("encryptKey", ""); - } else if (propertiesMap.get("encryptLevel").equals("0")) { - propertiesMap.put("encryptType", "org.apache.tsfile.encrypt.UNENCRYPTED"); - propertiesMap.put("encryptKey", ""); - } else if (propertiesMap.get("encryptLevel").equals("1")) { + if (!propertiesMap.containsKey("encryptLevel") || encryptLevel == null) { + propertiesMap.put("encryptLevel", stringToBytes("0")); + propertiesMap.put("encryptType", stringToBytes("org.apache.tsfile.encrypt.UNENCRYPTED")); + propertiesMap.put("encryptKey", stringToBytes("")); + } else if (encryptLevel.equals("0")) { + propertiesMap.put("encryptType", stringToBytes("org.apache.tsfile.encrypt.UNENCRYPTED")); + propertiesMap.put("encryptKey", stringToBytes("")); + } else if (encryptLevel.equals("1")) { if (!propertiesMap.containsKey("encryptType")) { throw new EncryptException( Messages.format("error.file.tsfile_metadata_no_encrypt_type", 1)); @@ -140,15 +147,15 @@ public static TsFileMetadata deserializeFrom( throw new EncryptException( Messages.format("error.file.tsfile_metadata_no_encrypt_key", 1)); } - if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) { + String encryptKey = getPropertyAsString(propertiesMap, "encryptKey"); + if (encryptKey == null || encryptKey.isEmpty()) { throw new EncryptException( Messages.format("error.file.tsfile_metadata_null_encrypt_key", 1)); } - String str = propertiesMap.get("encryptKey"); fileMetaData.encryptLevel = 1; - fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str); - fileMetaData.encryptType = propertiesMap.get("encryptType"); - } else if (propertiesMap.get("encryptLevel").equals("2")) { + fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(encryptKey); + fileMetaData.encryptType = getPropertyAsString(propertiesMap, "encryptType"); + } else if (encryptLevel.equals("2")) { if (!propertiesMap.containsKey("encryptType")) { throw new EncryptException( Messages.format("error.file.tsfile_metadata_no_encrypt_type", 2)); @@ -157,19 +164,17 @@ public static TsFileMetadata deserializeFrom( throw new EncryptException( Messages.format("error.file.tsfile_metadata_no_encrypt_key", 2)); } - if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) { + String encryptKey = getPropertyAsString(propertiesMap, "encryptKey"); + if (encryptKey == null || encryptKey.isEmpty()) { throw new EncryptException( Messages.format("error.file.tsfile_metadata_null_encrypt_key", 2)); } fileMetaData.encryptLevel = 2; - String str = propertiesMap.get("encryptKey"); - fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str); - fileMetaData.encryptType = propertiesMap.get("encryptType"); + fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(encryptKey); + fileMetaData.encryptType = getPropertyAsString(propertiesMap, "encryptType"); } else { throw new EncryptException( - Messages.format( - "error.file.tsfile_metadata_unsupported_encrypt_level", - propertiesMap.get("encryptLevel"))); + Messages.format("error.file.tsfile_metadata_unsupported_encrypt_level", encryptLevel)); } fileMetaData.tsFileProperties = propertiesMap; } @@ -177,7 +182,7 @@ public static TsFileMetadata deserializeFrom( return fileMetaData; } - public void addProperty(String key, String value) { + public void addProperty(String key, byte[] value) { if (tsFileProperties == null) { tsFileProperties = new HashMap<>(); } @@ -248,9 +253,15 @@ public int serializeTo(OutputStream outputStream) throws IOException { ReadWriteForEncodingUtils.writeVarInt( tsFileProperties != null ? tsFileProperties.size() : 0, outputStream); if (tsFileProperties != null) { - for (Entry entry : tsFileProperties.entrySet()) { + for (Entry entry : tsFileProperties.entrySet()) { byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream); - byteLen += ReadWriteIOUtils.writeVar(entry.getValue(), outputStream); + byte[] value = entry.getValue(); + byteLen += + ReadWriteForEncodingUtils.writeVarInt(value == null ? -1 : value.length, outputStream); + if (value != null) { + outputStream.write(value); + byteLen += value.length; + } } } @@ -295,7 +306,16 @@ public Map getTableSchemaMap() { return tableSchemaMap; } - public Map getTsFileProperties() { + public Map getTsFileProperties() { return tsFileProperties; } + + private static String getPropertyAsString(Map properties, String key) { + byte[] value = properties.get(key); + return value == null ? null : new String(value, TSFileConfig.STRING_CHARSET); + } + + private static byte[] stringToBytes(String value) { + return value.getBytes(TSFileConfig.STRING_CHARSET); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 505b6aa8a..9407eda65 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -608,7 +608,7 @@ public Map getTableSchemaMap() throws IOException { } /** Get the properties stored in the TsFile metadata. */ - public Map getTsFileProperties() throws IOException { + public Map getTsFileProperties() throws IOException { return readFileMetadata().getTsFileProperties(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java index 4dffbbc01..a3d125a1d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java @@ -80,7 +80,7 @@ public Optional getTableSchemas(String tableName) throws IOExceptio @Override @TsFileApi - public Map getTsFileProperties() throws IOException { + public Map getTsFileProperties() throws IOException { return fileReader.getTsFileProperties(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java index 0ade853c7..dd0c188e1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java @@ -50,7 +50,7 @@ ResultSet query( List getAllTableSchema() throws IOException; @TsFileApi - Map getTsFileProperties() throws IOException; + Map getTsFileProperties() throws IOException; @TsFileApi void close(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java index d0d73d59a..82b036bc8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java @@ -111,7 +111,7 @@ public static UpdateStatus updateTablePointCountIfMissing(File file) throws IOEx (tableName, pointCount) -> metadata.addProperty( TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, - Long.toString(pointCount))); + Long.toString(pointCount).getBytes(TSFileConfig.STRING_CHARSET))); rewriteFileMetadata(file.toPath(), fileMetadataPosition, metadata); return UpdateStatus.UPDATED; } @@ -176,14 +176,14 @@ private static TsFileSequenceReader openCompleteFile(File file) throws IOExcepti } private static boolean hasAllTablePointCountProperties( - Map properties, Set tableNames) { + Map properties, Set tableNames) { if (properties == null) { return false; } for (String tableName : tableNames) { - String value = properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName); + byte[] value = properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName); try { - if (value == null || Long.parseLong(value) < 0) { + if (value == null || Long.parseLong(new String(value, TSFileConfig.STRING_CHARSET)) < 0) { return false; } } catch (NumberFormatException e) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index 2bb39ad57..108de0051 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -261,7 +261,7 @@ protected Schema getSchema() { @Override @TsFileApi - public void addTsFileProperty(String key, String value) { + public void addTsFileProperty(String key, byte[] value) { fileWriter.addTsFileProperty(key, value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java index 15e5db30b..d20c432c5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/ITsFileWriter.java @@ -38,5 +38,5 @@ public interface ITsFileWriter extends AutoCloseable { void write(TSRecord record) throws IOException, WriteProcessException; @TsFileApi - void addTsFileProperty(String key, String value); + void addTsFileProperty(String key, byte[] value); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 79034b28e..3b2f148be 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -144,7 +144,7 @@ public class TsFileIOWriter implements AutoCloseable { protected Map tableSizeMap = new HashMap<>(); - private final Map tsFileProperties = new HashMap<>(); + private final Map tsFileProperties = new HashMap<>(); private final Map tablePointCountMap = new HashMap<>(); private boolean recordTablePointCount; @@ -261,7 +261,7 @@ public void setEncryptParam(EncryptParameter param) { } /** Add a custom property to the TsFile metadata. */ - public void addTsFileProperty(String key, String value) { + public void addTsFileProperty(String key, byte[] value) { tsFileProperties.put(key, value); } @@ -628,11 +628,13 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { tablePointCountMap.forEach( (tableName, pointCount) -> tsFileMetadata.addProperty( - TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, Long.toString(pointCount))); + TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, + Long.toString(pointCount).getBytes(TSFileConfig.STRING_CHARSET))); } - tsFileMetadata.addProperty("encryptLevel", encryptLevel); - tsFileMetadata.addProperty("encryptType", encryptType); - tsFileMetadata.addProperty("encryptKey", encryptKey); + tsFileMetadata.addProperty("encryptLevel", encryptLevel.getBytes(TSFileConfig.STRING_CHARSET)); + tsFileMetadata.addProperty("encryptType", encryptType.getBytes(TSFileConfig.STRING_CHARSET)); + tsFileMetadata.addProperty( + "encryptKey", encryptKey == null ? null : encryptKey.getBytes(TSFileConfig.STRING_CHARSET)); int size = tsFileMetadata.serializeTo(out.wrapAsStream()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java index dcf343918..6488acde9 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TsFileMetadataTest.java @@ -18,6 +18,7 @@ */ package org.apache.tsfile.file.metadata; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.compatibility.DeserializeConfig; import org.apache.tsfile.constant.TestConstant; import org.apache.tsfile.file.metadata.utils.TestHelper; @@ -53,10 +54,12 @@ public void tearDown() { @Test public void testWriteFileMetaData() { TsFileMetadata tsfMetaData = TestHelper.createSimpleFileMetaData(); - tsfMetaData.addProperty("encryptLevel", "0"); - tsfMetaData.addProperty("encryptType", "org.apache.tsfile.encrypt.UNENCRYPTED"); - tsfMetaData.addProperty("encryptKey", ""); - tsfMetaData.addProperty("d", "1"); + tsfMetaData.addProperty("encryptLevel", "0".getBytes(TSFileConfig.STRING_CHARSET)); + tsfMetaData.addProperty( + "encryptType", + "org.apache.tsfile.encrypt.UNENCRYPTED".getBytes(TSFileConfig.STRING_CHARSET)); + tsfMetaData.addProperty("encryptKey", "".getBytes(TSFileConfig.STRING_CHARSET)); + tsfMetaData.addProperty("d", new byte[] {0, 1, -1}); serialized(tsfMetaData); TsFileMetadata readMetaData = deSerialized(); Assert.assertTrue(Utils.isFileMetaDataEqual(tsfMetaData, readMetaData)); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java index dc37e00f9..27ec1ab24 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/utils/Utils.java @@ -25,7 +25,8 @@ import org.apache.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.tsfile.file.metadata.statistics.Statistics; -import java.util.Objects; +import java.util.Arrays; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -61,13 +62,29 @@ public static boolean isFileMetaDataEqual(TsFileMetadata metadata1, TsFileMetada MetadataIndexNode metaDataIndex2 = metadata2.getTableMetadataIndexNode(TestHelper.TEST_TABLE_NAME); - return Objects.equals(metadata1.getTsFileProperties(), metadata2.getTsFileProperties()) + return arePropertiesEqual(metadata1.getTsFileProperties(), metadata2.getTsFileProperties()) && metaDataIndex1.getChildren().size() == metaDataIndex2.getChildren().size(); } } return false; } + private static boolean arePropertiesEqual( + Map properties1, Map properties2) { + if (properties1 == null || properties2 == null) { + return properties1 == properties2; + } + if (!properties1.keySet().equals(properties2.keySet())) { + return false; + } + for (String key : properties1.keySet()) { + if (!Arrays.equals(properties1.get(key), properties2.get(key))) { + return false; + } + } + return true; + } + public static void isPageHeaderEqual(PageHeader header1, PageHeader header2) { if (Utils.isTwoObjectsNotNULL(header1, header2, "PageHeader")) { assertEquals(header1.getUncompressedSize(), header2.getUncompressedSize()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java index c8b0b5c0d..bbcc3278b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java @@ -42,6 +42,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -63,19 +64,24 @@ public void testTsFileProperties() throws IOException { try { try (ITsFileWriter writer = new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { - writer.addTsFileProperty("creator", "TsFileV4ReadWriteInterfacesTest"); - writer.addTsFileProperty("version", "1"); + writer.addTsFileProperty( + "creator", "TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8)); + writer.addTsFileProperty("version", "1".getBytes(StandardCharsets.UTF_8)); } try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { - Assert.assertEquals( - "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator")); - Assert.assertEquals("1", reader.getTsFileProperties().get("version")); + Assert.assertArrayEquals( + "TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8), + reader.getTsFileProperties().get("creator")); + Assert.assertArrayEquals( + "1".getBytes(StandardCharsets.UTF_8), reader.getTsFileProperties().get("version")); } try (ITsFileReader reader = new DeviceTableModelReader(file)) { - Assert.assertEquals( - "TsFileV4ReadWriteInterfacesTest", reader.getTsFileProperties().get("creator")); - Assert.assertEquals("1", reader.getTsFileProperties().get("version")); + Assert.assertArrayEquals( + "TsFileV4ReadWriteInterfacesTest".getBytes(StandardCharsets.UTF_8), + reader.getTsFileProperties().get("creator")); + Assert.assertArrayEquals( + "1".getBytes(StandardCharsets.UTF_8), reader.getTsFileProperties().get("version")); } } finally { Files.deleteIfExists(file.toPath()); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java index a1b604728..73341f992 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java @@ -34,6 +34,7 @@ import org.junit.Test; import java.io.File; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Arrays; import java.util.List; @@ -65,7 +66,7 @@ public void backfillMissingTablePointCountProperties() throws Exception { new ColumnSchema("s1", TSDataType.INT32, ColumnCategory.FIELD))); try (TsFileIOWriter ioWriter = new TsFileIOWriter(file); TsFileWriter writer = new TsFileWriter(ioWriter)) { - ioWriter.addTsFileProperty("custom.property", "preserved"); + ioWriter.addTsFileProperty("custom.property", "preserved".getBytes(StandardCharsets.UTF_8)); writer.registerTableSchema(tableSchema1); writer.registerTableSchema(tableSchema2); writer.writeTable(createTablet(tableSchema1, 3, true)); @@ -98,12 +99,15 @@ public Map> readChunkMetadataInDevice(IDeviceID devi try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { Assert.assertTrue(reader.isComplete()); - Map properties = reader.getTsFileProperties(); - Assert.assertEquals("preserved", properties.get("custom.property")); - Assert.assertEquals( - "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); - Assert.assertEquals( - "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); + Map properties = reader.getTsFileProperties(); + Assert.assertArrayEquals( + "preserved".getBytes(StandardCharsets.UTF_8), properties.get("custom.property")); + Assert.assertArrayEquals( + "5".getBytes(StandardCharsets.UTF_8), + properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); + Assert.assertArrayEquals( + "2".getBytes(StandardCharsets.UTF_8), + properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); Assert.assertEquals(2, reader.getAllDevices().size()); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java index e31c4fd44..3ed03c7be 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java @@ -161,12 +161,16 @@ private List createTablets( private void verifyProperties(File file, boolean enabled) throws IOException { try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { - Map properties = reader.getTsFileProperties(); + Map properties = reader.getTsFileProperties(); String table1Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1"; String table2Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2"; if (enabled) { - Assert.assertEquals(Long.toString(TABLE1_POINT_COUNT), properties.get(table1Key)); - Assert.assertEquals(Long.toString(TABLE2_POINT_COUNT), properties.get(table2Key)); + Assert.assertArrayEquals( + Long.toString(TABLE1_POINT_COUNT).getBytes(StandardCharsets.UTF_8), + properties.get(table1Key)); + Assert.assertArrayEquals( + Long.toString(TABLE2_POINT_COUNT).getBytes(StandardCharsets.UTF_8), + properties.get(table2Key)); } else { Assert.assertFalse(properties.containsKey(table1Key)); Assert.assertFalse(properties.containsKey(table2Key)); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index c8592afe1..b8caf3003 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -1290,11 +1290,13 @@ public void recordTablePointCountInProperties() throws IOException, WriteProcess } try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getAbsolutePath())) { - Map properties = reader.getTsFileProperties(); - Assert.assertEquals( - "5", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); - Assert.assertEquals( - "2", properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); + Map properties = reader.getTsFileProperties(); + Assert.assertArrayEquals( + "5".getBytes(StandardCharsets.UTF_8), + properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); + Assert.assertArrayEquals( + "2".getBytes(StandardCharsets.UTF_8), + properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); } } From a095b0704faf1623d698110acd05c7a37b72d554 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 30 Jul 2026 18:31:31 +0800 Subject: [PATCH 10/10] use byte[8] for table point count --- .../org/apache/tsfile/utils/TsFileTablePointCountTool.java | 6 +++--- .../org/apache/tsfile/write/writer/TsFileIOWriter.java | 2 +- .../apache/tsfile/utils/TsFileTablePointCountToolTest.java | 4 ++-- .../tsfile/write/TablePointCountPerformanceTest.java | 7 +++---- .../java/org/apache/tsfile/write/TsFileWriteApiTest.java | 5 +++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java index 82b036bc8..973c0215e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileTablePointCountTool.java @@ -111,7 +111,7 @@ public static UpdateStatus updateTablePointCountIfMissing(File file) throws IOEx (tableName, pointCount) -> metadata.addProperty( TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, - Long.toString(pointCount).getBytes(TSFileConfig.STRING_CHARSET))); + BytesUtils.longToBytes(pointCount))); rewriteFileMetadata(file.toPath(), fileMetadataPosition, metadata); return UpdateStatus.UPDATED; } @@ -183,10 +183,10 @@ private static boolean hasAllTablePointCountProperties( for (String tableName : tableNames) { byte[] value = properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName); try { - if (value == null || Long.parseLong(new String(value, TSFileConfig.STRING_CHARSET)) < 0) { + if (value == null || value.length != Long.BYTES || BytesUtils.bytesToLong(value) < 0) { return false; } - } catch (NumberFormatException e) { + } catch (RuntimeException e) { return false; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 3b2f148be..93ddec569 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -629,7 +629,7 @@ private void readChunkMetadataAndConstructIndexTree() throws IOException { (tableName, pointCount) -> tsFileMetadata.addProperty( TABLE_POINT_COUNT_PROPERTY_PREFIX + tableName, - Long.toString(pointCount).getBytes(TSFileConfig.STRING_CHARSET))); + BytesUtils.longToBytes(pointCount))); } tsFileMetadata.addProperty("encryptLevel", encryptLevel.getBytes(TSFileConfig.STRING_CHARSET)); tsFileMetadata.addProperty("encryptType", encryptType.getBytes(TSFileConfig.STRING_CHARSET)); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java index 73341f992..4133f031b 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TsFileTablePointCountToolTest.java @@ -103,10 +103,10 @@ public Map> readChunkMetadataInDevice(IDeviceID devi Assert.assertArrayEquals( "preserved".getBytes(StandardCharsets.UTF_8), properties.get("custom.property")); Assert.assertArrayEquals( - "5".getBytes(StandardCharsets.UTF_8), + BytesUtils.longToBytes(5), properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); Assert.assertArrayEquals( - "2".getBytes(StandardCharsets.UTF_8), + BytesUtils.longToBytes(2), properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); Assert.assertEquals(2, reader.getAllDevices().size()); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java index 3ed03c7be..786b36e13 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TablePointCountPerformanceTest.java @@ -24,6 +24,7 @@ import org.apache.tsfile.file.metadata.ColumnSchema; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -166,11 +167,9 @@ private void verifyProperties(File file, boolean enabled) throws IOException { String table2Key = TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2"; if (enabled) { Assert.assertArrayEquals( - Long.toString(TABLE1_POINT_COUNT).getBytes(StandardCharsets.UTF_8), - properties.get(table1Key)); + BytesUtils.longToBytes(TABLE1_POINT_COUNT), properties.get(table1Key)); Assert.assertArrayEquals( - Long.toString(TABLE2_POINT_COUNT).getBytes(StandardCharsets.UTF_8), - properties.get(table2Key)); + BytesUtils.longToBytes(TABLE2_POINT_COUNT), properties.get(table2Key)); } else { Assert.assertFalse(properties.containsKey(table1Key)); Assert.assertFalse(properties.containsKey(table2Key)); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index b8caf3003..4341da538 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -44,6 +44,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.read.v4.ITsFileReader; import org.apache.tsfile.read.v4.TsFileReaderBuilder; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; @@ -1292,10 +1293,10 @@ public void recordTablePointCountInProperties() throws IOException, WriteProcess try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getAbsolutePath())) { Map properties = reader.getTsFileProperties(); Assert.assertArrayEquals( - "5".getBytes(StandardCharsets.UTF_8), + BytesUtils.longToBytes(5), properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table1")); Assert.assertArrayEquals( - "2".getBytes(StandardCharsets.UTF_8), + BytesUtils.longToBytes(2), properties.get(TsFileIOWriter.TABLE_POINT_COUNT_PROPERTY_PREFIX + "table2")); } }