From 4e88c9bda580aa6448a6b9c6755c4744d77bc6c5 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Mon, 20 Jul 2026 23:46:27 +0800 Subject: [PATCH] ORC-2213: [C++] Add option to read timestamps with writer timezone Add a RowReaderOptions flag that lets timestamp readers use the writer timezone recorded in each stripe footer instead of the configured reader timezone. This preserves the stored timestamp value for callers that need to avoid ORC timestamp timezone conversion. --- c++/include/orc/Reader.hh | 10 ++++++++++ c++/src/Options.hh | 12 ++++++++++++ c++/src/Reader.cc | 7 +++++-- c++/src/Reader.hh | 1 + c++/test/TestWriter.cc | 38 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/c++/include/orc/Reader.hh b/c++/include/orc/Reader.hh index 122a781c93..4ae989ec58 100644 --- a/c++/include/orc/Reader.hh +++ b/c++/include/orc/Reader.hh @@ -353,6 +353,16 @@ namespace orc { */ const std::string& getTimezoneName() const; + /** + * Use the writer timezone when reading timestamp values. + */ + RowReaderOptions& setUseWriterTimezone(bool useWriterTimezone); + + /** + * Get whether to use the writer timezone when reading timestamp values. + */ + bool getUseWriterTimezone() const; + /** * Get the IdReadIntentMap map that was supplied by client. */ diff --git a/c++/src/Options.hh b/c++/src/Options.hh index c0abf190cc..18388780ce 100644 --- a/c++/src/Options.hh +++ b/c++/src/Options.hh @@ -151,6 +151,7 @@ namespace orc { bool enableLazyDecoding; std::shared_ptr sargs; std::string readerTimezone; + bool useWriterTimezone; RowReaderOptions::IdReadIntentMap idReadIntentMap; bool useTightNumericVector; std::shared_ptr readType; @@ -167,6 +168,7 @@ namespace orc { forcedScaleOnHive11Decimal = 6; enableLazyDecoding = false; readerTimezone = "GMT"; + useWriterTimezone = false; useTightNumericVector = false; throwOnSchemaEvolutionOverflow = false; enableAsyncPrefetch = false; @@ -318,6 +320,7 @@ namespace orc { RowReaderOptions& RowReaderOptions::setTimezoneName(const std::string& zoneName) { privateBits_->readerTimezone = zoneName; + privateBits_->useWriterTimezone = false; return *this; } @@ -325,6 +328,15 @@ namespace orc { return privateBits_->readerTimezone; } + RowReaderOptions& RowReaderOptions::setUseWriterTimezone(bool useWriterTimezone) { + privateBits_->useWriterTimezone = useWriterTimezone; + return *this; + } + + bool RowReaderOptions::getUseWriterTimezone() const { + return privateBits_->useWriterTimezone; + } + const RowReaderOptions::IdReadIntentMap RowReaderOptions::getIdReadIntentMap() const { return privateBits_->idReadIntentMap; } diff --git a/c++/src/Reader.cc b/c++/src/Reader.cc index c409576cc9..4c16b737fb 100644 --- a/c++/src/Reader.cc +++ b/c++/src/Reader.cc @@ -314,7 +314,9 @@ namespace orc { footer_(contents_->footer.get()), firstRowOfStripe_(*contents_->pool, 0), enableEncodedBlock_(opts.getEnableLazyDecoding()), - readerTimezone_(getTimezoneByName(opts.getTimezoneName())), + readerTimezone_(opts.getUseWriterTimezone() ? localTimezone_ + : getTimezoneByName(opts.getTimezoneName())), + useWriterTimezone_(opts.getUseWriterTimezone()), schemaEvolution_(opts.getReadType(), contents_->schema.get()) { uint64_t numberOfStripes; numberOfStripes = static_cast(footer_->stripes_size()); @@ -1366,7 +1368,8 @@ namespace orc { : localTimezone_; StripeStreamsImpl stripeStreams(*this, currentStripe_, currentStripeInfo_, currentStripeFooter_, currentStripeInfo_.offset(), - *contents_->stream, writerTimezone, readerTimezone_); + *contents_->stream, writerTimezone, + useWriterTimezone_ ? writerTimezone : readerTimezone_); reader_ = buildReader(*contents_->schema, stripeStreams, useTightNumericVector_, throwOnSchemaEvolutionOverflow_, /*convertToReadType=*/true); diff --git a/c++/src/Reader.hh b/c++/src/Reader.hh index 204f678e96..289e467930 100644 --- a/c++/src/Reader.hh +++ b/c++/src/Reader.hh @@ -189,6 +189,7 @@ namespace orc { // desired timezone to return data of timestamp types. const Timezone& readerTimezone_; + const bool useWriterTimezone_; // match read and file types SchemaEvolution schemaEvolution_; diff --git a/c++/test/TestWriter.cc b/c++/test/TestWriter.cc index f43e80ac8e..5827bcdbca 100644 --- a/c++/test/TestWriter.cc +++ b/c++/test/TestWriter.cc @@ -840,6 +840,44 @@ namespace orc { "2014-06-06 12:34:56", IS_DST); } + TEST_P(WriterTest, readTimestampWithWriterTimezone) { + MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE); + MemoryPool* pool = getDefaultPool(); + std::unique_ptr type(Type::buildTypeFromString("struct")); + + std::unique_ptr writer = + createWriter(16 * 1024, 64, 1024, CompressionKind_ZLIB, *type, pool, &memStream, + fileVersion, 0, "Asia/Shanghai"); + std::unique_ptr batch = writer->createRowBatch(1); + auto* structBatch = dynamic_cast(batch.get()); + auto* tsBatch = dynamic_cast(structBatch->fields[0]); + tsBatch->data[0] = 0; + tsBatch->nanoseconds[0] = 123000000; + structBatch->numElements = 1; + tsBatch->numElements = 1; + writer->add(*batch); + writer->close(); + + auto readTimestamp = [&](bool useWriterTimezone) { + auto inStream = + std::make_unique(memStream.getData(), memStream.getLength()); + std::unique_ptr reader = createReader(pool, std::move(inStream)); + RowReaderOptions rowReaderOptions; + rowReaderOptions.setTimezoneName("GMT"); + rowReaderOptions.setUseWriterTimezone(useWriterTimezone); + std::unique_ptr rowReader = reader->createRowReader(rowReaderOptions); + std::unique_ptr readBatch = rowReader->createRowBatch(1); + EXPECT_TRUE(rowReader->next(*readBatch)); + auto* readStructBatch = dynamic_cast(readBatch.get()); + auto* readTsBatch = dynamic_cast(readStructBatch->fields[0]); + EXPECT_EQ(123000000, readTsBatch->nanoseconds[0]); + return readTsBatch->data[0]; + }; + + EXPECT_EQ(8 * 60 * 60, readTimestamp(/*useWriterTimezone=*/false)); + EXPECT_EQ(0, readTimestamp(/*useWriterTimezone=*/true)); + } + // Test that the ORC-306 compensation (-1s for pre-1970 timestamps with // nanos > 999999) is applied BEFORE timezone conversion in the Reader. //