diff --git a/c++/include/orc/Reader.hh b/c++/include/orc/Reader.hh index 122a781c93..4ae989ec58 100644 --- a/c++/include/orc/Reader.hh +++ b/c++/include/orc/Reader.hh @@ -353,6 +353,16 @@ namespace orc { */ const std::string& getTimezoneName() const; + /** + * Use the writer timezone when reading timestamp values. + */ + RowReaderOptions& setUseWriterTimezone(bool useWriterTimezone); + + /** + * Get whether to use the writer timezone when reading timestamp values. + */ + bool getUseWriterTimezone() const; + /** * Get the IdReadIntentMap map that was supplied by client. */ diff --git a/c++/src/Options.hh b/c++/src/Options.hh index c0abf190cc..18388780ce 100644 --- a/c++/src/Options.hh +++ b/c++/src/Options.hh @@ -151,6 +151,7 @@ namespace orc { bool enableLazyDecoding; std::shared_ptr sargs; std::string readerTimezone; + bool useWriterTimezone; RowReaderOptions::IdReadIntentMap idReadIntentMap; bool useTightNumericVector; std::shared_ptr readType; @@ -167,6 +168,7 @@ namespace orc { forcedScaleOnHive11Decimal = 6; enableLazyDecoding = false; readerTimezone = "GMT"; + useWriterTimezone = false; useTightNumericVector = false; throwOnSchemaEvolutionOverflow = false; enableAsyncPrefetch = false; @@ -318,6 +320,7 @@ namespace orc { RowReaderOptions& RowReaderOptions::setTimezoneName(const std::string& zoneName) { privateBits_->readerTimezone = zoneName; + privateBits_->useWriterTimezone = false; return *this; } @@ -325,6 +328,15 @@ namespace orc { return privateBits_->readerTimezone; } + RowReaderOptions& RowReaderOptions::setUseWriterTimezone(bool useWriterTimezone) { + privateBits_->useWriterTimezone = useWriterTimezone; + return *this; + } + + bool RowReaderOptions::getUseWriterTimezone() const { + return privateBits_->useWriterTimezone; + } + const RowReaderOptions::IdReadIntentMap RowReaderOptions::getIdReadIntentMap() const { return privateBits_->idReadIntentMap; } diff --git a/c++/src/Reader.cc b/c++/src/Reader.cc index c409576cc9..4c16b737fb 100644 --- a/c++/src/Reader.cc +++ b/c++/src/Reader.cc @@ -314,7 +314,9 @@ namespace orc { footer_(contents_->footer.get()), firstRowOfStripe_(*contents_->pool, 0), enableEncodedBlock_(opts.getEnableLazyDecoding()), - readerTimezone_(getTimezoneByName(opts.getTimezoneName())), + readerTimezone_(opts.getUseWriterTimezone() ? localTimezone_ + : getTimezoneByName(opts.getTimezoneName())), + useWriterTimezone_(opts.getUseWriterTimezone()), schemaEvolution_(opts.getReadType(), contents_->schema.get()) { uint64_t numberOfStripes; numberOfStripes = static_cast(footer_->stripes_size()); @@ -1366,7 +1368,8 @@ namespace orc { : localTimezone_; StripeStreamsImpl stripeStreams(*this, currentStripe_, currentStripeInfo_, currentStripeFooter_, currentStripeInfo_.offset(), - *contents_->stream, writerTimezone, readerTimezone_); + *contents_->stream, writerTimezone, + useWriterTimezone_ ? writerTimezone : readerTimezone_); reader_ = buildReader(*contents_->schema, stripeStreams, useTightNumericVector_, throwOnSchemaEvolutionOverflow_, /*convertToReadType=*/true); diff --git a/c++/src/Reader.hh b/c++/src/Reader.hh index 204f678e96..289e467930 100644 --- a/c++/src/Reader.hh +++ b/c++/src/Reader.hh @@ -189,6 +189,7 @@ namespace orc { // desired timezone to return data of timestamp types. const Timezone& readerTimezone_; + const bool useWriterTimezone_; // match read and file types SchemaEvolution schemaEvolution_; diff --git a/c++/test/TestWriter.cc b/c++/test/TestWriter.cc index f43e80ac8e..5827bcdbca 100644 --- a/c++/test/TestWriter.cc +++ b/c++/test/TestWriter.cc @@ -840,6 +840,44 @@ namespace orc { "2014-06-06 12:34:56", IS_DST); } + TEST_P(WriterTest, readTimestampWithWriterTimezone) { + MemoryOutputStream memStream(DEFAULT_MEM_STREAM_SIZE); + MemoryPool* pool = getDefaultPool(); + std::unique_ptr type(Type::buildTypeFromString("struct")); + + std::unique_ptr writer = + createWriter(16 * 1024, 64, 1024, CompressionKind_ZLIB, *type, pool, &memStream, + fileVersion, 0, "Asia/Shanghai"); + std::unique_ptr batch = writer->createRowBatch(1); + auto* structBatch = dynamic_cast(batch.get()); + auto* tsBatch = dynamic_cast(structBatch->fields[0]); + tsBatch->data[0] = 0; + tsBatch->nanoseconds[0] = 123000000; + structBatch->numElements = 1; + tsBatch->numElements = 1; + writer->add(*batch); + writer->close(); + + auto readTimestamp = [&](bool useWriterTimezone) { + auto inStream = + std::make_unique(memStream.getData(), memStream.getLength()); + std::unique_ptr reader = createReader(pool, std::move(inStream)); + RowReaderOptions rowReaderOptions; + rowReaderOptions.setTimezoneName("GMT"); + rowReaderOptions.setUseWriterTimezone(useWriterTimezone); + std::unique_ptr rowReader = reader->createRowReader(rowReaderOptions); + std::unique_ptr readBatch = rowReader->createRowBatch(1); + EXPECT_TRUE(rowReader->next(*readBatch)); + auto* readStructBatch = dynamic_cast(readBatch.get()); + auto* readTsBatch = dynamic_cast(readStructBatch->fields[0]); + EXPECT_EQ(123000000, readTsBatch->nanoseconds[0]); + return readTsBatch->data[0]; + }; + + EXPECT_EQ(8 * 60 * 60, readTimestamp(/*useWriterTimezone=*/false)); + EXPECT_EQ(0, readTimestamp(/*useWriterTimezone=*/true)); + } + // Test that the ORC-306 compensation (-1s for pre-1970 timestamps with // nanos > 999999) is applied BEFORE timezone conversion in the Reader. //