From afbbb04150c64417011062460e9379cb36c6ef32 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 13 Jul 2026 17:11:37 +0100 Subject: [PATCH 1/6] New C++ API Signed-off-by: Mikhail Kot --- .clang-format | 2 +- .github/workflows/ci.yml | 36 +- docs/Doxyfile.cpp | 28 ++ docs/conf.py | 27 -- lang/cpp/CMakeLists.txt | 112 ++++++ lang/cpp/README.md | 37 ++ lang/cpp/include/vortex/array.hpp | 434 ++++++++++++++++++++++++ lang/cpp/include/vortex/common.hpp | 98 ++++++ lang/cpp/include/vortex/data_source.hpp | 59 ++++ lang/cpp/include/vortex/dtype.hpp | 189 +++++++++++ lang/cpp/include/vortex/error.hpp | 55 +++ lang/cpp/include/vortex/estimate.hpp | 47 +++ lang/cpp/include/vortex/expression.hpp | 182 ++++++++++ lang/cpp/include/vortex/scalar.hpp | 76 +++++ lang/cpp/include/vortex/scan.hpp | 248 ++++++++++++++ lang/cpp/include/vortex/session.hpp | 35 ++ lang/cpp/include/vortex/writer.hpp | 52 +++ lang/cpp/src/array.cpp | 358 +++++++++++++++++++ lang/cpp/src/data_source.cpp | 112 ++++++ lang/cpp/src/dtype.cpp | 225 ++++++++++++ lang/cpp/src/expression.cpp | 159 +++++++++ lang/cpp/src/f16.cpp | 40 +++ lang/cpp/src/scalar.cpp | 128 +++++++ lang/cpp/src/scan.cpp | 98 ++++++ lang/cpp/src/session.cpp | 27 ++ lang/cpp/src/writer.cpp | 50 +++ 26 files changed, 2885 insertions(+), 29 deletions(-) create mode 100644 docs/Doxyfile.cpp create mode 100644 lang/cpp/CMakeLists.txt create mode 100644 lang/cpp/README.md create mode 100644 lang/cpp/include/vortex/array.hpp create mode 100644 lang/cpp/include/vortex/common.hpp create mode 100644 lang/cpp/include/vortex/data_source.hpp create mode 100644 lang/cpp/include/vortex/dtype.hpp create mode 100644 lang/cpp/include/vortex/error.hpp create mode 100644 lang/cpp/include/vortex/estimate.hpp create mode 100644 lang/cpp/include/vortex/expression.hpp create mode 100644 lang/cpp/include/vortex/scalar.hpp create mode 100644 lang/cpp/include/vortex/scan.hpp create mode 100644 lang/cpp/include/vortex/session.hpp create mode 100644 lang/cpp/include/vortex/writer.hpp create mode 100644 lang/cpp/src/array.cpp create mode 100644 lang/cpp/src/data_source.cpp create mode 100644 lang/cpp/src/dtype.cpp create mode 100644 lang/cpp/src/expression.cpp create mode 100644 lang/cpp/src/f16.cpp create mode 100644 lang/cpp/src/scalar.cpp create mode 100644 lang/cpp/src/scan.cpp create mode 100644 lang/cpp/src/session.cpp create mode 100644 lang/cpp/src/writer.cpp diff --git a/.clang-format b/.clang-format index 587f746da78..bb77abc1478 100644 --- a/.clang-format +++ b/.clang-format @@ -25,7 +25,7 @@ DerivePointerAlignment: false IncludeBlocks: Regroup InsertBraces: true Language: Cpp -NamespaceIndentation: Inner +NamespaceIndentation: None PointerAlignment: Right SpaceBeforeCpp11BracedList: true SpaceBeforeCtorInitializerColon: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b694bf364d..e7aae5dc647 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,7 +355,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: C/C++ Lint - clang-format run: | - git ls-files vortex-cuda vortex-duckdb vortex-ffi \ + git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \ | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ | grep -v 'arrow/reference/arrow_c_device\.h$' \ | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ @@ -522,6 +522,40 @@ jobs: with: command: check ${{ matrix.checks }} + cxx-api: + name: "C++ API" + timeout-minutes: 5 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - name: Install Rust nightly toolchain + run: | + rustup toolchain install $NIGHTLY_TOOLCHAIN + rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview + - name: Build FFI library (asan) + run: | + RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ + -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ + -Zsanitizer=address,leak" \ + cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ + --target x86_64-unknown-linux-gnu -Zbuild-std \ + -p vortex-ffi + - name: Build C++ library (asan) + run: | + mkdir -p build + cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" + cmake --build build --parallel $(nproc) + sqllogic-test: name: "SQL logic tests" needs: duckdb-ready diff --git a/docs/Doxyfile.cpp b/docs/Doxyfile.cpp new file mode 100644 index 00000000000..23072f7479a --- /dev/null +++ b/docs/Doxyfile.cpp @@ -0,0 +1,28 @@ +# Doxygen configuration for Vortex C++ API documentation. +# XML output is consumed by Sphinx via the Breathe extension. + +PROJECT_NAME = "Vortex C++" +OUTPUT_DIRECTORY = _build/doxygen-cpp + +# Input sources +INPUT = ../lang/cpp/include/vortex +FILE_PATTERNS = *.hpp +RECURSIVE = NO + +# We only care about XML output for Breathe +GENERATE_XML = YES +GENERATE_HTML = NO +GENERATE_LATEX = NO +XML_PROGRAMLISTING = YES + +# Extract everything, even if not fully documented yet +EXTRACT_ALL = YES +EXTRACT_PRIVATE = NO +EXTRACT_STATIC = YES + +# Preprocessing — resolve includes but don't expand macros +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO + +# Suppress warnings about undocumented members (WIP API) +WARN_IF_UNDOCUMENTED = NO diff --git a/docs/conf.py b/docs/conf.py index f795fea0d6b..a2cadbb255f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -125,33 +125,6 @@ breathe_projects = {"vortex-cpp": _doxygen_xml_dir} breathe_default_project = "vortex-cpp" -# C++ types from cxx bridge and standard library that Sphinx cannot resolve. -nitpick_ignore += [ - ("cpp:identifier", t) - for t in [ - "vortex", - "rust", - "ffi", - "uint8_t", - "uint16_t", - "uint32_t", - "uint64_t", - "int8_t", - "int16_t", - "int32_t", - "int64_t", - "size_t", - "std::size_t", - ] -] -nitpick_ignore_regex = [ - # cxx bridge internals that will never be resolvable in Sphinx. - (r"cpp:identifier", r"rust::.*"), - (r"cpp:identifier", r"ffi::.*"), - # Doxygen file-level labels (e.g. "dtype_8hpp") that we don't generate pages for. - (r"ref", r".*_8hpp"), -] - # -- Options for hawkmoth C API gen ---------------------------- hawkmoth_root = str(git_root / "vortex-ffi/cinclude") diff --git a/lang/cpp/CMakeLists.txt b/lang/cpp/CMakeLists.txt new file mode 100644 index 00000000000..c2a792080a0 --- /dev/null +++ b/lang/cpp/CMakeLists.txt @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +cmake_minimum_required(VERSION 3.10) +project(VortexCXX + VERSION 0.0.1 + LANGUAGES CXX) + +include(FetchContent) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_program(SCCACHE_PROGRAM sccache) +if (SCCACHE_PROGRAM) + set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") + message(STATUS "Sccache found: ${SCCACHE_PROGRAM}") +else () + message(STATUS "Sccache not found") +endif () + +set(SANITIZER "" CACHE STRING "Build with sanitizers") +set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") +set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library") + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() + +if (NOT SANITIZER STREQUAL "") + message(NOTICE "Sanitizer: ${SANITIZER}") + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(FATAL_ERROR "Only debug build is supported for sanitizer builds") + endif() + + if (SANITIZER STREQUAL "asan") + set(SANITIZER_FLAGS "-fsanitize=address,undefined,leak") + elseif (SANITIZER STREQUAL "tsan") + set(SANITIZER_FLAGS "-fsanitize=thread") + else() + message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}") + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZER_FLAGS}") +endif() + +if (RUST_BUILD_PROFILE STREQUAL "") + if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + set(RUST_BUILD_PROFILE "release_debug") + else() + string(TOLOWER "${CMAKE_BUILD_TYPE}" RUST_BUILD_PROFILE) + endif() +endif() + +set(VORTEX_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../vortex-ffi") +set(VORTEX_FFI_LIB_DIR "${VORTEX_FFI_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}") +set(VORTEX_FFI_HEADERS "${VORTEX_FFI_DIR}/cinclude") + +if(APPLE) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dylib") +elseif(WIN32) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.lib") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dll") +else() + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.so") +endif() + +if(NOT EXISTS "${VORTEX_FFI_STATIC}") + message(FATAL_ERROR + "vortex-ffi static library not found at ${VORTEX_FFI_STATIC}. " + "Run: cargo build --profile -p vortex-ffi") +endif() + +add_library(vortex_ffi STATIC IMPORTED) +set_target_properties(vortex_ffi PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_STATIC}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}") + +if(EXISTS "${VORTEX_FFI_SHARED}") + add_library(vortex_ffi_shared SHARED IMPORTED) + set_target_properties(vortex_ffi_shared PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_SHARED}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}" + INTERFACE_LINK_OPTIONS "LINKER:-rpath,${VORTEX_FFI_LIB_DIR}") +endif() + +file(GLOB VORTEX_CXX_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_library(vortex_cxx STATIC ${VORTEX_CXX_SOURCES}) +target_include_directories(vortex_cxx PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude +) +target_link_libraries(vortex_cxx PUBLIC vortex_ffi) +add_library(vortex::cxx ALIAS vortex_cxx) + +if(TARGET vortex_ffi_shared) + add_library(vortex_cxx_shared SHARED ${VORTEX_CXX_SOURCES}) + set_target_properties(vortex_cxx_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(vortex_cxx_shared PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude + ) + target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared) + add_library(vortex::cxx_shared ALIAS vortex_cxx_shared) +endif() diff --git a/lang/cpp/README.md b/lang/cpp/README.md new file mode 100644 index 00000000000..a0b92f21835 --- /dev/null +++ b/lang/cpp/README.md @@ -0,0 +1,37 @@ +# Vortex C++ bindings + +For a usage guide, see docs/api/cpp/index.rst. + +## Requirements + +- CMake 3.10+ +- C++20 compiler (C++23 compiler for tests). +- Rust toolchain for building `vortex-ffi`. + +## Build + +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DCMAKE_BUILD_TYPE=Release +cmake --build build -j +``` + +This will generate `libvortex_cxx` shared and static libraries. +You can use `vortex_cxx` and `vortex_cxx_shared` CMake targets. + +## Test + +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DBUILD_TESTS=ON +cmake --build build -j +ctest --test-dir build -j "$(nproc)" +``` + +## Run examples + +```sh +cmake -Bbuild -DBUILD_EXAMPLES=ON +cmake --build build -j +./build/examples/hello-vortex +``` diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp new file mode 100644 index 00000000000..ad06b1eb7d5 --- /dev/null +++ b/lang/cpp/include/vortex/array.hpp @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/expression.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace vortex { + +// Types that a PrimitiveView can hold +template +concept primitive_view = primitive_type || std::is_same_v; + +template +class PrimitiveView; + +class Array; +class StringView; +class BytesView; + +enum class ValidityType { + // Items can't be null + NonNullable = VX_VALIDITY_NON_NULLABLE, + // All items are valid + AllValid = VX_VALIDITY_ALL_VALID, + // All items are invalid + AllInvalid = VX_VALIDITY_ALL_INVALID, + // Item validity is set in a boolean array: true = valid, false = invalid + Array = VX_VALIDITY_ARRAY, +}; + +constexpr ValidityType NonNullable = ValidityType::NonNullable; +constexpr ValidityType AllValid = ValidityType::AllValid; +constexpr ValidityType AllInvalid = ValidityType::AllInvalid; + +/** + * Array per-element validity of type ValidityType. + * If ValidityType is ValidityType::Array, holds a boolean array + * with validity items. + * + * You can use shortcut constants NonNullable/AllValid/AllInvalid and + * function ValidityArray(validity_bools); + */ +class Validity { +public: + // NonNullable/AllValid/AllInvalid constructor + // NOLINTNEXTLINE(google-explicit-constructor) + Validity(ValidityType type); + Validity(ValidityType type, const Array &array); + + Validity(const Validity &other); + Validity(Validity &&other) noexcept; + Validity &operator=(const Validity &other); + Validity &operator=(Validity &&other) noexcept; + ~Validity(); + + ValidityType type() const noexcept { + return type_; + } + + // Boolean validity array. Throws if type() != ValidityType::Array. + Array array() const; + +private: + friend struct detail::Access; + friend Validity ValidityArray(const Array &bools); + Validity(ValidityType type, const vx_array *owned) : type_(type), array_(owned) { + } + + ValidityType type_; + const vx_array *array_; +}; + +// Validity determined by a boolean array, true = valid, false = invalid. +Validity ValidityArray(const Array &bools); + +namespace detail { +// Validity bitmap for typed views. Owns the arrays that back the bits +class ValidityBits { +public: + ValidityBits(ValidityBits &&other) noexcept; + ValidityBits &operator=(ValidityBits &&other) noexcept; + ValidityBits(const ValidityBits &) = delete; + ValidityBits &operator=(const ValidityBits &) = delete; + ~ValidityBits(); + + bool is_null(size_t index) const noexcept; + +private: + friend class vortex::Array; + // Materialize validity of "canonical" + ValidityBits(const Session &session, const vx_array *canonical); + + const vx_array *owner_ = nullptr; + const uint8_t *bits_ = nullptr; + size_t bit_offset_ = 0; + bool all_invalid_ = false; +}; +} // namespace detail + +// A reference-counted handle to columnar data in some encoding +class Array { +public: + Array(const Array &other); + Array(Array &&) noexcept = default; + Array &operator=(const Array &other); + Array &operator=(Array &&) noexcept = default; + + // An all-null array with DataType Null. + static Array null(size_t len); + + /** + * A Primitive array copied from a typed buffer. + * + * Example: + * + * std::array buffer = {0, 1, 2}; + * auto array = Array::primitive(buffer); + */ + template + static Array primitive(std::span data, const Validity &validity = NonNullable) { + return primitive_raw(detail::to_ptype(), data.data(), data.size(), validity); + } + + /** + * Import an Arrow array. Consumes both "array" and "schema", do not use + * or release them afterwards. For a record batch pass nullable = false. + */ + static Array from_arrow(ArrowArray *array, ArrowSchema *schema, bool nullable); + + size_t size() const; + bool nullable() const; + bool has_dtype(DataTypeVariant variant) const; + bool is_primitive(PType ptype) const; + DataType dtype() const; + Validity validity() const; + + // Number of null/invalid elements in Array + size_t null_count() const; + + /** + * Get a Struct field by index. Throws if Array is not a Struct or if index + * is out of bounds. + */ + Array field(size_t index) const; + + /** + * Get a Struct field by name. Throws if Array is not a Struct or doesn't + * have this named field. + */ + Array field(std::string_view name) const; + + /* + * Create a new Array slicing [begin; end) rows from original. + * Doesn't copy the original buffer or sliced buffer. + * + * Example: + * + * std::array buffer = {0, 1, 2}; + * Array array = Array::primitive(buffer); + * Array sliced = array.slice(1, 2); + */ + Array slice(size_t begin, size_t end) const; + + /** + * Apply an expression to an array. + * + * This function operates in constant time and doesn't execute the result + * array. To execute the array, canonicalise it. + * + * Example: + * + * using namespace vortex::expr::ops; + * + * std::array buffer = {0, 1, 2}; + * Array array = Array::primitive(buffer); + * Expression expr = expr::root() > expr::lit(0); + * Array result = array.apply(expr); + */ + Array apply(const Expression &expr) const; + + /** + * Bulk view over values. Canonicalizes the array. + * Throws if T does not match Array's ptype. + * + * Example: + * + * Session session; + * std::array buffer = {0, 1, 2}; + * Array array = Array::primitive(buffer); + * auto view = array.values(session); + */ + template + PrimitiveView values(const Session &session) const; + + // Bulk view over Bool values + PrimitiveView bools(const Session &session) const; + + // Bulk view over Utf8 values. + StringView strings(const Session &session) const; + + // Bulk view over Binary values. + BytesView bytes(const Session &session) const; + +private: + friend struct detail::Access; + friend class StringView; + friend class BytesView; + template + friend class PrimitiveView; + + explicit Array(const vx_array *owned) : handle_(owned) { + } + const vx_array *release() && noexcept { + return handle_.release(); + } + + static Array primitive_raw(vx_ptype ptype, const void *data, size_t len, const Validity &validity); + Array canonicalize(const Session &session) const; + + struct Deleter { + void operator()(const vx_array *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +/** + * Create a Struct array from named columns of equal length. + * + * Example: + * + * std::array age_buffer = {0, 1, 2}; + * std::array height_buffer = {0, 1, 2}; + * Array ages = Array::primitive(age_buffer); + * Array heights = Array::primitive(height_buffer); + * Array result = make_struct( + * {{"age", ages}, {"height", heights}}, + * NonNullable); + */ +Array make_struct(std::initializer_list> fields, + const Validity &validity = NonNullable); + +/** + * Create a Struct Array from a dynamic number of fields. + * Prefer make_struct() if you know all fields beforehand. + * + * Example: + * + * std::array age_buffer = {0, 1, 2}; + * std::array height_buffer = {0, 1, 2}; + * Array ages = Array::primitive(age_buffer); + * Array heights = Array::primitive(height_buffer); + * + * StructArrayBuilder b(NonNullable, 2); + * Array result = builder + * .add("age", ages) + * .add("height", heights) + * .build(); + */ +class StructArrayBuilder { +public: + explicit StructArrayBuilder(const Validity &validity, size_t capacity = 0); + StructArrayBuilder(const StructArrayBuilder &) = delete; + StructArrayBuilder &operator=(const StructArrayBuilder &) = delete; + StructArrayBuilder(StructArrayBuilder &&) noexcept = default; + StructArrayBuilder &operator=(StructArrayBuilder &&) noexcept = default; + + StructArrayBuilder &add(std::string_view name, const Array &field) &; + StructArrayBuilder &&add(std::string_view name, const Array &field) &&; + + // Consume the builder and return and Array + Array build() &&; + +private: + struct Deleter { + void operator()(vx_struct_column_builder *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +/** + * Typed read-only view over a Primitive array. + * + * Owns a canonicalized copy of Array. values() and anything derived from + * it are valid as long as the view lives. + */ +template +class PrimitiveView { +public: + /* + * Get raw values from this view. Values at null/invalid positions are + * unspecified. + */ + std::span values() const noexcept { + return {data_, size_}; + } + bool is_null(size_t index) const noexcept { + return validity_.is_null(index); + } + size_t size() const noexcept { + return size_; + } + +private: + friend class Array; + PrimitiveView(Array canonical, detail::ValidityBits validity, const T *data, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), data_(data), size_(size) { + } + + Array canonical_; + detail::ValidityBits validity_; + const T *data_; + size_t size_; +}; + +/** + * Read-only view over a Bool array. As Bool values are bit-packed, there's no + * span. Read individual values with value(i). + */ +template <> +class PrimitiveView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + bool value(size_t index) const; + bool is_null(size_t index) const noexcept { + return validity_.is_null(index); + } + size_t size() const noexcept { + return size_; + } + +private: + friend class Array; + PrimitiveView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) { + } + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +/** + * Read-only view over a Utf8 array. + * + * operator[] is O(1) and borrows from the view's canonical copy. Returned + * string_views are valid as long as the view lives. + */ +class StringView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + std::string_view operator[](size_t index) const; + bool is_null(size_t index) const noexcept { + return validity_.is_null(index); + } + size_t size() const noexcept { + return size_; + } + +private: + friend class Array; + StringView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) { + } + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +/** + * Read-only view over a Binary array. + * + * Byte spans borrow from the view's canonical copy and are valid as long as + * the view lives. + */ +class BytesView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + BinaryView operator[](size_t index) const; + bool is_null(size_t index) const noexcept { + return validity_.is_null(index); + } + size_t size() const noexcept { + return size_; + } + +private: + friend class Array; + BytesView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) { + } + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +template +PrimitiveView Array::values(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = detail::Access::c_ptr(canonical); + if (!vx_array_is_primitive(raw, detail::to_ptype())) { + throw VortexException("values: T does not match the array's ptype", ErrorCode::MismatchedTypes); + } + vx_error *error = nullptr; + const void *data = vx_array_data_ptr_primitive(raw, &error); + detail::throw_on_error(error); + detail::ValidityBits validity(session, raw); + const size_t n = vx_array_len(raw); + return PrimitiveView(std::move(canonical), std::move(validity), static_cast(data), n); +} +} // namespace vortex diff --git a/lang/cpp/include/vortex/common.hpp b/lang/cpp/include/vortex/common.hpp new file mode 100644 index 00000000000..cf70888f7ce --- /dev/null +++ b/lang/cpp/include/vortex/common.hpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include +#include +#include +#include +#include + +#if __STDCPP_FLOAT16_T__ != 1 + +namespace vortex { +struct float16_t { + uint16_t bits; + friend bool operator==(float16_t, float16_t) = default; + // NOLINTNEXTLINE + operator float() const; +}; +static_assert(sizeof(float16_t) == 2 && std::is_trivially_copyable_v); +} // namespace vortex + +#else + +#include + +namespace vortex { +using float16_t = ::std::float16_t; +} // namespace vortex + +#endif + +namespace vortex::detail { +struct Access { + template + static T adopt(Args &&...args) { + return T(std::forward(args)...); + } + template + static auto release(T &&t) noexcept { + return std::forward(t).release(); + } + template + static auto c_ptr(const T &t) noexcept { + return t.handle_.get(); + } +}; + +template +constexpr vx_ptype to_ptype() { + if constexpr (std::is_same_v) { + return PTYPE_U8; + } else if constexpr (std::is_same_v) { + return PTYPE_U16; + } else if constexpr (std::is_same_v) { + return PTYPE_U32; + } else if constexpr (std::is_same_v) { + return PTYPE_U64; + } else if constexpr (std::is_same_v) { + return PTYPE_I8; + } else if constexpr (std::is_same_v) { + return PTYPE_I16; + } else if constexpr (std::is_same_v) { + return PTYPE_I32; + } else if constexpr (std::is_same_v) { + return PTYPE_I64; + } else if constexpr (std::is_same_v) { + return PTYPE_F16; + } else if constexpr (std::is_same_v) { + return PTYPE_F32; + } else { + static_assert(std::is_same_v); + return PTYPE_F64; + } +} + +template +inline constexpr bool is_numeric_element = + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v; +} // namespace vortex::detail + +namespace vortex { + +// View over a single Binary byte range. +using BinaryView = std::span; + +// Types that can be stored in a Primitive array +template +concept primitive_type = detail::is_numeric_element; + +// Types constructible as scalars or literals. +template +concept element_type = primitive_type || std::is_same_v || std::is_same_v || + std::is_same_v; +} // namespace vortex diff --git a/lang/cpp/include/vortex/data_source.hpp b/lang/cpp/include/vortex/data_source.hpp new file mode 100644 index 00000000000..af99090601f --- /dev/null +++ b/lang/cpp/include/vortex/data_source.hpp @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/estimate.hpp" +#include "vortex/scan.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace vortex { + +/** + * A reference to one or more (possibly remote, possibly glob) paths. + * + * Creating a DataSource opens the first matched path to read the schema. All + * other IO is deferred until a scan is requested. Multiple scans may be + * requested from one data source. + */ +class DataSource { +public: + static DataSource open(const Session &session, std::initializer_list paths); + static DataSource open(const Session &session, std::span paths); + static DataSource open(const Session &session, std::span paths); + + /** + * Create a DataSource from an in-memory Vortex file. Borrows the buffer: + * caller must keep it alive and unmodified while DataSource is alive. + */ + static DataSource from_buffer(const Session &session, std::span data); + + DataSource(const DataSource &other); + DataSource(DataSource &&) noexcept = default; + DataSource &operator=(const DataSource &other); + DataSource &operator=(DataSource &&) noexcept = default; + + Estimate row_count() const; + DataType dtype() const; + Scan scan(ScanOptions options = {}) const; + +private: + friend struct detail::Access; + DataSource(const vx_data_source *owned, Session session) : handle_(owned), session_(std::move(session)) { + } + + struct Deleter { + void operator()(const vx_data_source *ptr) const noexcept; + }; + std::unique_ptr handle_; + Session session_; +}; +} // namespace vortex diff --git a/lang/cpp/include/vortex/dtype.hpp b/lang/cpp/include/vortex/dtype.hpp new file mode 100644 index 00000000000..edb62dd839d --- /dev/null +++ b/lang/cpp/include/vortex/dtype.hpp @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace vortex { + +struct Field; + +enum class DataTypeVariant { + Null = DTYPE_NULL, + Bool = DTYPE_BOOL, + // Primitives e.g., u8, i16, f32 + Primitive = DTYPE_PRIMITIVE, + // Variable-length UTF-8 string + Utf8 = DTYPE_UTF8, + // Variable-length binary + Binary = DTYPE_BINARY, + // Nested struct + Struct = DTYPE_STRUCT, + // Nested list + List = DTYPE_LIST, + // User-defined extension + Extension = DTYPE_EXTENSION, + // Decimal with fixed precision and scale + Decimal = DTYPE_DECIMAL, + // Nested fixed-size list + FixedSizeList = DTYPE_FIXED_SIZE_LIST, +}; + +// Primitive type +enum class PType { + U8 = PTYPE_U8, + U16 = PTYPE_U16, + U32 = PTYPE_U32, + U64 = PTYPE_U64, + I8 = PTYPE_I8, + I16 = PTYPE_I16, + I32 = PTYPE_I32, + I64 = PTYPE_I64, + F16 = PTYPE_F16, + F32 = PTYPE_F32, + F64 = PTYPE_F64, +}; + +/** + * A Vortex data type. Data types are logical: they say nothing about physical + * representation. + */ +class DataType { +public: + DataType(const DataType &other); + DataType(DataType &&) noexcept = default; + DataType &operator=(const DataType &other); + DataType &operator=(DataType &&) noexcept = default; + + /** + * Consume an ArrowSchema and convert it into a DataType. The schema must + * not be used after this call. + */ + static DataType from_arrow(ArrowSchema *schema); + + /** + * Convert dtype to an Arrow C schema. Caller is responsible for invoking + * schema's release() callback. + */ + ArrowSchema to_arrow() const; + + DataTypeVariant variant() const; + bool nullable() const; + + PType primitive_type() const; + uint8_t decimal_precision() const; + int8_t decimal_scale() const; + + /** + * For a Struct dtype, return its fields in order. + * Throws if DataType is not Struct. + */ + std::vector fields() const; + + // List accessors. Valid only on List and FixedSizeList dtypes + + DataType list_element() const; + DataType fixed_size_list_element() const; + uint32_t fixed_size_list_size() const; + +private: + friend struct detail::Access; + explicit DataType(const vx_dtype *owned); + const vx_dtype *release() && noexcept { + return handle_.release(); + } + + struct Deleter { + void operator()(const vx_dtype *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +// Field of a Struct dtype +struct Field { + std::string name; + DataType dtype; +}; + +namespace dtype { + +inline constexpr bool Nullable = true; + +DataType null(); +DataType boolean(bool nullable = false); +DataType primitive(PType ptype, bool nullable = false); +DataType int8(bool nullable = false); +DataType int16(bool nullable = false); +DataType int32(bool nullable = false); +DataType int64(bool nullable = false); +DataType uint8(bool nullable = false); +DataType uint16(bool nullable = false); +DataType uint32(bool nullable = false); +DataType uint64(bool nullable = false); +DataType float16(bool nullable = false); +DataType float32(bool nullable = false); +DataType float64(bool nullable = false); +DataType utf8(bool nullable = false); +DataType binary(bool nullable = false); +DataType decimal(uint8_t precision, int8_t scale, bool nullable = false); +DataType list(DataType element, bool nullable = false); +DataType fixed_size_list(DataType element, uint32_t size, bool nullable = false); + +/** + * Create a DataTypeVariant::Struct from a field list. + * + * Example: + * + * using dtype::Nullable; + * DataType dtype = dtype::struct_({ + * {"age", dtype::uint8()}, + * {"height", dtype::uint16(Nullable)}} + * ); + */ +DataType struct_(std::initializer_list> fields, bool nullable = false); +} // namespace dtype + +/* + * Create a DataTypeVariant::Struct dynamically. + * Prefer dtype::struct_ if fields are known beforehand. + * + * Example: + * + * using dtype::Nullable; + * DataType dtype = StructFieldsBuilder() + * .add("age", dtype::uint8()) + * .add("height", dtype::uint16(Nullable)}) + * .build(Nullable); + * ); + */ +class StructFieldsBuilder { +public: + StructFieldsBuilder(); + StructFieldsBuilder(const StructFieldsBuilder &) = delete; + StructFieldsBuilder &operator=(const StructFieldsBuilder &) = delete; + StructFieldsBuilder(StructFieldsBuilder &&) noexcept = default; + StructFieldsBuilder &operator=(StructFieldsBuilder &&) noexcept = default; + + StructFieldsBuilder &add(std::string_view name, const DataType &dtype) &; + StructFieldsBuilder &&add(std::string_view name, const DataType &dtype) &&; + + // Consume the builder and return a DataType + DataType build(bool nullable = false) &&; + +private: + struct Deleter { + void operator()(vx_struct_fields_builder *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; +} // namespace vortex diff --git a/lang/cpp/include/vortex/error.hpp b/lang/cpp/include/vortex/error.hpp new file mode 100644 index 00000000000..ba76b5c9385 --- /dev/null +++ b/lang/cpp/include/vortex/error.hpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include + +#include +#include +#include + +namespace vortex { +enum class ErrorCode { + Other = VX_ERROR_CODE_OTHER, + OutOfBounds = VX_ERROR_CODE_OUT_OF_BOUNDS, + Compute = VX_ERROR_CODE_COMPUTE, + InvalidArgument = VX_ERROR_CODE_INVALID_ARGUMENT, + Serialization = VX_ERROR_CODE_SERIALIZATION, + NotImplemented = VX_ERROR_CODE_NOT_IMPLEMENTED, + MismatchedTypes = VX_ERROR_CODE_MISMATCHED_TYPES, + AssertionFailed = VX_ERROR_CODE_ASSERTION_FAILED, + Io = VX_ERROR_CODE_IO, + Panic = VX_ERROR_CODE_PANIC, +}; + +class VortexException : public std::runtime_error { +public: + VortexException(const std::string &message, ErrorCode code) : std::runtime_error(message), code_(code) { + } + + ErrorCode code() const noexcept { + return code_; + } + +private: + ErrorCode code_; +}; + +namespace detail { +// Throw VortexException and free "error" if it is non-nullptr. +inline void throw_on_error(vx_error *error) { + if (error == nullptr) { + return; + } + const vx_view str = vx_error_message(error); + const std::string message {str.ptr, str.len}; + const auto code = static_cast(vx_error_get_code(error)); + vx_error_free(error); + throw VortexException(message, code); +} + +inline vx_view to_view(std::string_view view) noexcept { + return {view.data(), view.size()}; +} +} // namespace detail +} // namespace vortex diff --git a/lang/cpp/include/vortex/estimate.hpp b/lang/cpp/include/vortex/estimate.hpp new file mode 100644 index 00000000000..19fc174bf5c --- /dev/null +++ b/lang/cpp/include/vortex/estimate.hpp @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/error.hpp" +#include + +#include + +namespace vortex { + +enum class EstimateType { + Unknown = VX_ESTIMATE_UNKNOWN, + Exact = VX_ESTIMATE_EXACT, + Inexact = VX_ESTIMATE_INEXACT, +}; + +// Estimated count (rows in a partition, partitions in a scan) +class Estimate { +public: + explicit Estimate(vx_estimate raw) : raw_(raw) { + } + + inline EstimateType type() const noexcept { + return static_cast(raw_.type); + } + + /** + * Estimated count. Throws if type() is Unknown. For inexact estimates this + * is an upper bound. + */ + inline uint64_t value() const { + if (type() == EstimateType::Unknown) { + throw VortexException("estimate is unknown", ErrorCode::InvalidArgument); + } + return raw_.estimate; + } + + inline uint64_t value_or(uint64_t fallback) const noexcept { + return type() == EstimateType::Unknown ? fallback : raw_.estimate; + } + +private: + vx_estimate raw_; +}; + +} // namespace vortex diff --git a/lang/cpp/include/vortex/expression.hpp b/lang/cpp/include/vortex/expression.hpp new file mode 100644 index 00000000000..86486557e21 --- /dev/null +++ b/lang/cpp/include/vortex/expression.hpp @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/scalar.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +// A node in an expression tree used for scan filters and projections. +class Expression { +public: + Expression(const Expression &); + Expression &operator=(const Expression &); + Expression(Expression &&) noexcept = default; + Expression &operator=(Expression &&) noexcept = default; + + /** + * Extract field from a Struct. Output DataType is field's DataType. + * + * Errors at scan/apply time if field does not exist or if root() is + * not a Struct. + * + * Example: + * + * Expression field = root()["age"]["nested"]; + */ + Expression operator[](std::string_view field) const; + + Expression is_null() const; + + /* + * Extract fields from a Struct. Output DataType is a Struct. + * + * Errors at scan/apply time if any of fields does not exist or if root() + * is not a Struct. + */ + Expression select(std::span names) const; + Expression select(std::span names) const; + Expression select(std::initializer_list names) const; + +private: + friend struct detail::Access; + explicit Expression(const vx_expression *owned) : handle_(owned) { + } + const vx_expression *release() && noexcept { + return handle_.release(); + } + + struct Deleter { + void operator()(const vx_expression *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +// Add, Sub, Mul, and Div error in runtime on overflow and underflow +enum class BinaryOperator { + // x == y + Eq = VX_OPERATOR_EQ, + // x != y + NotEq = VX_OPERATOR_NOT_EQ, + // x > y + Gt = VX_OPERATOR_GT, + // x >= y + Gte = VX_OPERATOR_GTE, + // x < y + Lt = VX_OPERATOR_LT, + // x <= y + Lte = VX_OPERATOR_LTE, + // boolean x AND y, Kleene semantics + KleeneAnd = VX_OPERATOR_KLEENE_AND, + // boolean x OR y, Kleene semantics + KleeneOr = VX_OPERATOR_KLEENE_OR, + // x + y + Add = VX_OPERATOR_ADD, + // x - y + Sub = VX_OPERATOR_SUB, + // x * y + Mul = VX_OPERATOR_MUL, + // x / y + Div = VX_OPERATOR_DIV, +}; + +namespace expr { + +// scanned/applied array. +Expression root(); + +// root()'s named column. +Expression col(std::string_view name); + +// Literal expression +Expression lit(const Scalar &value); + +/* + * Literal expression. + * + * Literal's DataType must match column it's compared against, otherwise scan + * fails at runtime. No type coercion is performed. + */ +template +Expression lit(T value) { + return lit(scalar::of(value)); +} + +Expression eq(const Expression &l, const Expression &r); +Expression neq(const Expression &l, const Expression &r); +Expression lt(const Expression &l, const Expression &r); +Expression lte(const Expression &l, const Expression &r); +Expression gt(const Expression &l, const Expression &r); +Expression gte(const Expression &l, const Expression &r); +Expression add(const Expression &l, const Expression &r); +Expression sub(const Expression &l, const Expression &r); +Expression mul(const Expression &l, const Expression &r); +Expression div(const Expression &l, const Expression &r); +Expression binary_op(BinaryOperator op, const Expression &l, const Expression &r); + +// Kleene AND of children. Errors on an empty list. +Expression and_all(std::span children); +// Kleene OR of children. Errors on an empty list. +Expression or_all(std::span children); + +Expression logical_not(const Expression &child); +Expression is_null(const Expression &child); +Expression list_contains(const Expression &list, const Expression &value); + +/** + * Opt-in operator overloads like in Eigen. + * Note && and || don't short-circuit. + */ +namespace ops { + +inline Expression operator==(const Expression &l, const Expression &r) { + return eq(l, r); +} +inline Expression operator!=(const Expression &l, const Expression &r) { + return neq(l, r); +} +inline Expression operator<(const Expression &l, const Expression &r) { + return lt(l, r); +} +inline Expression operator<=(const Expression &l, const Expression &r) { + return lte(l, r); +} +inline Expression operator>(const Expression &l, const Expression &r) { + return gt(l, r); +} +inline Expression operator>=(const Expression &l, const Expression &r) { + return gte(l, r); +} +inline Expression operator&&(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::KleeneAnd, l, r); +} +inline Expression operator||(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::KleeneOr, l, r); +} +inline Expression operator!(const Expression &e) { + return logical_not(e); +} +inline Expression operator+(const Expression &l, const Expression &r) { + return add(l, r); +} +inline Expression operator-(const Expression &l, const Expression &r) { + return sub(l, r); +} +inline Expression operator*(const Expression &l, const Expression &r) { + return mul(l, r); +} +inline Expression operator/(const Expression &l, const Expression &r) { + return div(l, r); +} + +} // namespace ops +} // namespace expr +} // namespace vortex diff --git a/lang/cpp/include/vortex/scalar.hpp b/lang/cpp/include/vortex/scalar.hpp new file mode 100644 index 00000000000..02dc006b93b --- /dev/null +++ b/lang/cpp/include/vortex/scalar.hpp @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +// A single value with an associated DataType +class Scalar { +public: + Scalar(const Scalar &other); + Scalar(Scalar &&) noexcept = default; + Scalar &operator=(const Scalar &other); + Scalar &operator=(Scalar &&) noexcept = default; + + bool is_null() const; + DataType dtype() const; + +private: + friend struct detail::Access; + explicit Scalar(vx_scalar *owned) : handle_(owned) { + } + vx_scalar *release() && noexcept { + return handle_.release(); + } + + struct Deleter { + void operator()(vx_scalar *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +namespace detail { +vx_scalar *make_bool(bool value, bool nullable); +vx_scalar *make_primitive(vx_ptype ptype, const void *value, bool nullable); +vx_scalar *make_utf8(std::string_view value, bool nullable); +vx_scalar *make_binary(BinaryView value, bool nullable); +Scalar adopt(vx_scalar *raw); +} // namespace detail + +namespace scalar { +/** + * A scalar of DataType selected by T: bool, primitive, string_view (utf8), + * or a BinaryView (binary). + * + * Bytes are copied for utf8 and binary scalars. + */ +template +Scalar of(T value, bool nullable = false) { + if constexpr (std::is_same_v) { + return detail::adopt(detail::make_bool(value, nullable)); + } else if constexpr (std::is_same_v) { + return detail::adopt(detail::make_utf8(value, nullable)); + } else if constexpr (std::is_same_v) { + return detail::adopt(detail::make_binary(value, nullable)); + } else { + return detail::adopt(detail::make_primitive(detail::to_ptype(), &value, nullable)); + } +} + +// A typed null of (a nullable copy of) a given DataType. +Scalar null(const DataType &dtype); + +Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable = false); +Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable = false); +} // namespace scalar +} // namespace vortex diff --git a/lang/cpp/include/vortex/scan.hpp b/lang/cpp/include/vortex/scan.hpp new file mode 100644 index 00000000000..9113ce1d00a --- /dev/null +++ b/lang/cpp/include/vortex/scan.hpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/estimate.hpp" +#include "vortex/expression.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace vortex { + +namespace detail { +// range-for support for Scan and Partition +template +class PullRange { +public: + class iterator { + public: + using value_type = Item; + using difference_type = std::ptrdiff_t; + + iterator() = default; + iterator(Source *src, std::optional first) : src_(src), cur_(std::move(first)) { + } + + Item &operator*() const { + return *cur_; + } + iterator &operator++() { + cur_ = src_->pull(); + return *this; + } + void operator++(int) { + ++*this; + } + bool operator==(std::default_sentinel_t) const { + return !cur_.has_value(); + } + + private: + Source *src_ = nullptr; + mutable std::optional cur_; + }; + + explicit PullRange(Source &src) : src_(&src) { + } + iterator begin() { + return iterator(src_, src_->pull()); + } + std::default_sentinel_t end() { + return std::default_sentinel; + } + +private: + Source *src_; +}; +} // namespace detail + +// Wrapper around ArrowArrayStream which releases stream in destructor +class ArrowStream { +public: + ArrowStream(const ArrowStream &) = delete; + ArrowStream &operator=(const ArrowStream &) = delete; + ArrowStream(ArrowStream &&other) noexcept; + ArrowStream &operator=(ArrowStream &&other) noexcept; + ~ArrowStream(); + + ArrowArrayStream *raw() noexcept { + return &stream_; + } + +private: + friend struct detail::Access; + ArrowStream(Session session, ArrowArrayStream stream) noexcept + : session_(std::move(session)), stream_(stream) { + } + + Session session_; + ArrowArrayStream stream_ {}; +}; + +/** + * An independent unit of scan work. + * + * Partition's methods are thread-unsafe: drive each partition from + * one worker thread. + * Calling methods of a moved-out Partition is UB. + */ +class Partition { +public: + Partition(const Partition &) = delete; + Partition &operator=(const Partition &) = delete; + Partition(Partition &&) noexcept = default; + Partition &operator=(Partition &&) noexcept = default; + + // Estimated row count. Throws if called after next(). + Estimate row_count() const; + + // Return next Array or nullopt when partition is exhausted. + std::optional next(); + + // range-for over Arrays + auto batches() & { + return detail::PullRange(*this); + } + auto batches() && = delete; + + /* + * Consume the partition into an Arrow stream. + * Blocks until partition is drained. + */ + ArrowStream into_arrow_stream() &&; + +private: + friend struct detail::Access; + friend class detail::PullRange; + Partition(vx_partition *owned, Session session) : handle_(owned), session_(std::move(session)) { + } + + std::optional pull() { + return next(); + } + + struct Deleter { + void operator()(vx_partition *ptr) const noexcept; + }; + std::unique_ptr handle_; + Session session_; +}; + +/* + * Row range [begin; end) to apply over filtering. + * [0; 0) or convenience constant AllRows means "return all rows". + */ +struct RowRange { + uint64_t begin = 0; + uint64_t end = 0; +}; + +// Return all rows +constexpr RowRange AllRows = RowRange {0, 0}; + +struct Selection { + enum class Kind { + Include = VX_SELECTION_INCLUDE_RANGE, + Exclude = VX_SELECTION_EXCLUDE_RANGE, + }; + Kind kind = Kind::Include; + std::vector indices; +}; + +/** + * Scan configuration. Fields are append-only and must be set via designated + * initializers. + * Default fields have reasonable behaviour: default projection returns all + * fields, default filter, row_range, and selection don't filter etc. + * + * Example: + * + * DataSource ds = DataSource::open(session, {"file.vortex"}); + * Scan scan = ds.scan({.limit = 100}); + */ +struct ScanOptions { + std::optional projection; + std::optional filter; + /* + * Row range [begin; end) to apply over filtering. + * [0; 0) or convenience constant AllRows means "return all rows". + */ + std::optional row_range; + // Row-index filter applied after row_range. + std::optional selection; + /* + * Maximum number of rows to return. 0 means no limit. + * You can either pass a limit or a filter but not both. + */ + uint64_t limit = 0; + // If true, return rows in storage order. + bool ordered = false; +}; + +/** + * A single traversal of a DataSource. A scan can be consumed only once. + * + * next_partition() is internally synchronized, give each partition to its own + * worker thread. + * + * Calling methods of a moved-out Scan is UB. + */ +class Scan { +public: + Scan(const Scan &) = delete; + Scan &operator=(const Scan &) = delete; + Scan(Scan &&) noexcept = default; + Scan &operator=(Scan &&) noexcept = default; + + Estimate partition_count() const noexcept { + return estimate_; + } + + /** + * Scan's dtype. + * + * Throws if called after next_partition(). + * UB if called in parallel with next_partition(). + */ + DataType dtype() const; + + // Next partition or nullopt when the scan is exhausted. Thread-safe. + std::optional next_partition(); + + // range-for over partitions + auto partitions() & { + return detail::PullRange(*this); + } + auto partitions() && = delete; + +private: + friend struct detail::Access; + friend class detail::PullRange; + Scan(vx_scan *owned, Estimate estimate, Session session) + : handle_(owned), mutex_(std::make_unique()), estimate_(estimate), + session_(std::move(session)) { + } + + std::optional pull() { + return next_partition(); + } + + struct Deleter { + void operator()(vx_scan *ptr) const noexcept; + }; + std::unique_ptr handle_; + std::unique_ptr mutex_; + Estimate estimate_; + Session session_; +}; +} // namespace vortex diff --git a/lang/cpp/include/vortex/session.hpp b/lang/cpp/include/vortex/session.hpp new file mode 100644 index 00000000000..29522e7fd74 --- /dev/null +++ b/lang/cpp/include/vortex/session.hpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" + +#include + +#include + +namespace vortex { + +/** + * A handle to a Vortex session, registry of encodings and compute kernels. + * Copying shares the underlying session. + */ +class Session { +public: + Session(); + + Session(const Session &other); + Session(Session &&) noexcept = default; + Session &operator=(const Session &other); + Session &operator=(Session &&) noexcept = default; + +private: + friend struct detail::Access; + + struct Deleter { + void operator()(vx_session *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +} // namespace vortex diff --git a/lang/cpp/include/vortex/writer.hpp b/lang/cpp/include/vortex/writer.hpp new file mode 100644 index 00000000000..da761c334f9 --- /dev/null +++ b/lang/cpp/include/vortex/writer.hpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/array.hpp" +#include "vortex/dtype.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include + +namespace vortex { + +/** + * Writes arrays into a Vortex file. + * + * finish() writes the footer and finalizes the file. + * Not calling finish() leaves file corrupted. + */ +class Writer { +public: + static Writer open(const Session &session, std::string_view path, const DataType &dtype); + + Writer(const Writer &) = delete; + Writer &operator=(const Writer &) = delete; + Writer(Writer &&) noexcept = default; + Writer &operator=(Writer &&) noexcept = default; + + /* + * Append Array to output file. + * Throws if "array"'s DataType doesn't match writer's DataType. + */ + void push(const Array &array); + + /* + * Write footer and finalize the file. + * Throws on failure. Writer is closed afterwards and further uses throws. + */ + void finish(); + +private: + explicit Writer(vx_array_sink *sink) : handle_(sink) { + } + + struct Deleter { + void operator()(vx_array_sink *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; +} // namespace vortex diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp new file mode 100644 index 00000000000..b0177319857 --- /dev/null +++ b/lang/cpp/src/array.cpp @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; + +Validity::Validity(const Validity &other) + : type_(other.type_), array_(other.array_ != nullptr ? vx_array_clone(other.array_) : nullptr) { +} + +Validity::Validity(ValidityType type) : type_(type), array_(nullptr) { + if (type == ValidityType::Array) { + throw VortexException("Validity(ValidityType) called with ValidityType::Array", + ErrorCode::InvalidArgument); + } +} + +Validity::Validity(ValidityType type, const Array &array) + : type_(type), array_(vx_array_clone(Access::c_ptr(array))) { +} + +Validity::Validity(Validity &&other) noexcept : type_(other.type_), array_(other.array_) { + other.array_ = nullptr; + other.type_ = ValidityType::NonNullable; +} + +Validity &Validity::operator=(const Validity &other) { + if (this != &other) { + *this = Validity(other); + } + return *this; +} + +Validity &Validity::operator=(Validity &&other) noexcept { + if (this != &other) { + vx_array_free(array_); + type_ = other.type_; + array_ = other.array_; + other.array_ = nullptr; + other.type_ = ValidityType::NonNullable; + } + return *this; +} + +Validity::~Validity() { + vx_array_free(array_); +} + +Array Validity::array() const { + if (type_ != ValidityType::Array || array_ == nullptr) { + throw VortexException("validity has no backing array", ErrorCode::InvalidArgument); + } + return Access::adopt(vx_array_clone(array_)); +} + +Validity ValidityArray(const Array &bools) { + return Validity(ValidityType::Array, vx_array_clone(Access::c_ptr(bools))); +} + +bool detail::ValidityBits::is_null(size_t index) const noexcept { + if (all_invalid_) { + return true; + } + if (bits_ == nullptr) { + return false; + } + const size_t bit = bit_offset_ + index; + return (bits_[bit / 8] >> (bit % 8) & 1) == 0; +} + +namespace detail { +ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) { + vx_validity raw {}; + vx_error *error = nullptr; + vx_array_get_validity(canonical, &raw, &error); + throw_on_error(error); + + switch (static_cast(raw.type)) { + case ValidityType::NonNullable: + case ValidityType::AllValid: + return; + case ValidityType::AllInvalid: + all_invalid_ = true; + return; + case ValidityType::Array: + break; + } + + owner_ = vx_array_canonicalize(Access::c_ptr(session), raw.array, &error); + vx_array_free(raw.array); + throw_on_error(error); + + bits_ = static_cast(vx_array_data_ptr_bool(owner_, &bit_offset_, &error)); + if (error != nullptr) { + vx_array_free(owner_); + } + throw_on_error(error); +} + +ValidityBits::ValidityBits(ValidityBits &&other) noexcept + : owner_(other.owner_), bits_(other.bits_), bit_offset_(other.bit_offset_), + all_invalid_(other.all_invalid_) { + other.owner_ = nullptr; + other.bits_ = nullptr; +} + +ValidityBits &ValidityBits::operator=(ValidityBits &&other) noexcept { + if (this != &other) { + vx_array_free(owner_); + owner_ = other.owner_; + bits_ = other.bits_; + bit_offset_ = other.bit_offset_; + all_invalid_ = other.all_invalid_; + other.owner_ = nullptr; + other.bits_ = nullptr; + } + return *this; +} + +ValidityBits::~ValidityBits() { + vx_array_free(owner_); +} +} // namespace detail + +static const vx_struct_fields *struct_fields_or_throw(const vx_dtype *dtype) { + const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); + if (fields == nullptr) { + throw VortexException("dtype is not a struct", ErrorCode::MismatchedTypes); + } + return fields; +} + +void Array::Deleter::operator()(const vx_array *ptr) const noexcept { + vx_array_free(ptr); +} + +Array::Array(const Array &other) : handle_(vx_array_clone(other.handle_.get())) { +} + +Array &Array::operator=(const Array &other) { + if (this != &other) { + handle_.reset(vx_array_clone(other.handle_.get())); + } + return *this; +} + +Array Array::null(size_t len) { + return Access::adopt(vx_array_new_null(len)); +} + +Array Array::primitive_raw(vx_ptype ptype, const void *data, size_t len, const Validity &validity) { + std::optional keep_alive; + vx_validity raw {}; + raw.type = static_cast(validity.type()); + if (validity.type() == ValidityType::Array) { + keep_alive = validity.array(); + raw.array = Access::c_ptr(*keep_alive); + } + + vx_error *error = nullptr; + const vx_array *out = vx_array_new_primitive(ptype, data, len, &raw, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::from_arrow(ArrowArray *array, ArrowSchema *schema, bool nullable) { + vx_error *error = nullptr; + const vx_array *out = vx_array_from_arrow(array, schema, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +size_t Array::size() const { + return vx_array_len(handle_.get()); +} + +bool Array::nullable() const { + return vx_array_is_nullable(handle_.get()); +} + +bool Array::has_dtype(DataTypeVariant v) const { + return vx_array_has_dtype(handle_.get(), static_cast(v)); +} + +bool Array::is_primitive(PType p) const { + return vx_array_is_primitive(handle_.get(), static_cast(p)); +} + +DataType Array::dtype() const { + return Access::adopt(vx_array_dtype(handle_.get())); +} + +Validity Array::validity() const { + vx_validity raw {}; + vx_error *error = nullptr; + vx_array_get_validity(handle_.get(), &raw, &error); + throw_on_error(error); + return Access::adopt(static_cast(raw.type), raw.array); +} + +size_t Array::null_count() const { + vx_error *error = nullptr; + const size_t count = vx_array_invalid_count(handle_.get(), &error); + throw_on_error(error); + return count; +} + +Array Array::field(size_t index) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_get_field(handle_.get(), index, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::field(std::string_view name) const { + const DataType dt = dtype(); + const std::unique_ptr fields( + struct_fields_or_throw(detail::Access::c_ptr(dt)), + &vx_struct_fields_free); + const uint64_t fields_size = vx_struct_fields_nfields(fields.get()); + for (uint64_t i = 0; i < fields_size; ++i) { + const vx_view field = vx_struct_fields_field_name(fields.get(), i); + if (std::string_view {field.ptr, field.len} == name) { + return this->field(i); + } + } + throw VortexException("no field named \"" + std::string(name) + "\"", ErrorCode::InvalidArgument); +} + +Array Array::slice(size_t begin, size_t end) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_slice(handle_.get(), begin, end, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::apply(const Expression &expr) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_apply(handle_.get(), Access::c_ptr(expr), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::canonicalize(const Session &session) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_canonicalize(Access::c_ptr(session), handle_.get(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array make_struct(std::initializer_list> fields, + const Validity &validity) { + StructArrayBuilder builder(validity, fields.size()); + for (const auto &[name, field] : fields) { + builder.add(name, field); + } + return std::move(builder).build(); +} + +void StructArrayBuilder::Deleter::operator()(vx_struct_column_builder *ptr) const noexcept { + vx_struct_column_builder_free(ptr); +} + +StructArrayBuilder::StructArrayBuilder(const Validity &validity, size_t capacity) { + std::optional keep_alive; + vx_validity raw {}; + raw.type = static_cast(validity.type()); + if (validity.type() == ValidityType::Array) { + keep_alive = validity.array(); + raw.array = Access::c_ptr(*keep_alive); + } + handle_.reset(vx_struct_column_builder_new(&raw, capacity)); +} + +StructArrayBuilder &StructArrayBuilder::add(std::string_view name, const Array &field) & { + vx_error *error = nullptr; + vx_struct_column_builder_add_field(handle_.get(), detail::to_view(name), Access::c_ptr(field), &error); + throw_on_error(error); + return *this; +} + +StructArrayBuilder &&StructArrayBuilder::add(std::string_view name, const Array &field) && { + add(name, field); + return std::move(*this); +} + +Array StructArrayBuilder::build() && { + vx_error *error = nullptr; + const vx_array *out = vx_struct_column_builder_finalize(handle_.release(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +PrimitiveView Array::bools(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_BOOL)) { + throw VortexException("bools(): array is not a Bool array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return PrimitiveView(std::move(canonical), std::move(validity), len); +} + +StringView Array::strings(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_UTF8)) { + throw VortexException("strings(): array is not a Utf8 array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return StringView(std::move(canonical), std::move(validity), len); +} + +BytesView Array::bytes(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_BINARY)) { + throw VortexException("bytes(): array is not a Binary array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return BytesView(std::move(canonical), std::move(validity), len); +} + +bool PrimitiveView::value(size_t i) const { + return vx_array_get_bool(Access::c_ptr(canonical_), i); +} + +std::string_view StringView::operator[](size_t i) const { + vx_error *error = nullptr; + const vx_view out = vx_array_utf8_at(Access::c_ptr(canonical_), i, &error); + throw_on_error(error); + return {out.ptr, out.len}; +} + +BinaryView BytesView::operator[](size_t i) const { + vx_error *error = nullptr; + const vx_view out = vx_array_binary_at(Access::c_ptr(canonical_), i, &error); + throw_on_error(error); + return {reinterpret_cast(out.ptr), out.len}; +} +} // namespace vortex diff --git a/lang/cpp/src/data_source.cpp b/lang/cpp/src/data_source.cpp new file mode 100644 index 00000000000..f032f47128d --- /dev/null +++ b/lang/cpp/src/data_source.cpp @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/data_source.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void DataSource::Deleter::operator()(const vx_data_source *ptr) const noexcept { + vx_data_source_free(ptr); +} + +DataSource::DataSource(const DataSource &other) + : handle_(vx_data_source_clone(other.handle_.get())), session_(other.session_) { +} + +DataSource &DataSource::operator=(const DataSource &other) { + if (this != &other) { + handle_.reset(vx_data_source_clone(other.handle_.get())); + session_ = other.session_; + } + return *this; +} + +template +static DataSource open_paths(const Session &session, std::span paths) { + std::vector raw; + raw.reserve(paths.size()); + for (const auto &path : paths) { + raw.push_back(to_view(path)); + } + + vx_data_source_options options {}; + options.paths = raw.data(); + options.paths_len = raw.size(); + + vx_error *error = nullptr; + const vx_data_source *ds = vx_data_source_new(Access::c_ptr(session), &options, &error); + throw_on_error(error); + return Access::adopt(ds, session); +} + +DataSource DataSource::open(const Session &session, std::initializer_list paths) { + std::span span {paths.begin(), paths.end()}; + return open_paths(session, span); +} + +DataSource DataSource::open(const Session &session, std::span paths) { + return open_paths(session, paths); +} + +DataSource DataSource::open(const Session &session, std::span paths) { + return open_paths(session, paths); +} + +DataSource DataSource::from_buffer(const Session &session, std::span data) { + vx_error *error = nullptr; + const vx_data_source *ds = + vx_data_source_new_buffer(Access::c_ptr(session), data.data(), data.size(), &error); + throw_on_error(error); + return Access::adopt(ds, session); +} + +Estimate DataSource::row_count() const { + vx_estimate raw {}; + vx_data_source_get_row_count(handle_.get(), &raw); + return Estimate(raw); +} + +DataType DataSource::dtype() const { + return Access::adopt(vx_data_source_dtype(handle_.get())); +} + +Scan DataSource::scan(ScanOptions options) const { + vx_scan_options raw {}; + raw.projection = options.projection.has_value() ? Access::c_ptr(*options.projection) : nullptr; + raw.filter = options.filter.has_value() ? Access::c_ptr(*options.filter) : nullptr; + if (options.row_range.has_value()) { + raw.row_range_begin = options.row_range->begin; + raw.row_range_end = options.row_range->end; + } + if (options.selection.has_value()) { + raw.selection.idx = options.selection->indices.data(); + raw.selection.idx_len = options.selection->indices.size(); + raw.selection.include = static_cast(options.selection->kind); + } else { + raw.selection.include = VX_SELECTION_INCLUDE_ALL; + } + raw.limit = options.limit; + raw.ordered = options.ordered; + + vx_estimate estimate {}; + vx_error *error = nullptr; + vx_scan *scan = vx_data_source_scan(handle_.get(), &raw, &estimate, &error); + throw_on_error(error); + return Access::adopt(scan, Estimate(estimate), session_); +} + +} // namespace vortex diff --git a/lang/cpp/src/dtype.cpp b/lang/cpp/src/dtype.cpp new file mode 100644 index 00000000000..c5f5c7947bc --- /dev/null +++ b/lang/cpp/src/dtype.cpp @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; +using namespace std::string_literals; + +void DataType::Deleter::operator()(const vx_dtype *ptr) const noexcept { + vx_dtype_free(ptr); +} + +DataType::DataType(const vx_dtype *owned) : handle_(owned) { +} + +DataType::DataType(const DataType &other) : handle_(vx_dtype_clone(other.handle_.get())) { +} + +DataType &DataType::operator=(const DataType &other) { + if (this != &other) { + handle_.reset(vx_dtype_clone(other.handle_.get())); + } + return *this; +} + +DataType DataType::from_arrow(ArrowSchema *schema) { + vx_error *error = nullptr; + const vx_dtype *dtype = vx_dtype_from_arrow_schema(schema, &error); + throw_on_error(error); + return DataType(dtype); +} + +ArrowSchema DataType::to_arrow() const { + ArrowSchema schema {}; + vx_error *error = nullptr; + vx_dtype_to_arrow_schema(handle_.get(), &schema, &error); + throw_on_error(error); + return schema; +} + +DataTypeVariant DataType::variant() const { + return static_cast(vx_dtype_get_variant(handle_.get())); +} + +bool DataType::nullable() const { + return vx_dtype_is_nullable(handle_.get()); +} + +PType DataType::primitive_type() const { + return static_cast(vx_dtype_primitive_ptype(handle_.get())); +} + +uint8_t DataType::decimal_precision() const { + return vx_dtype_decimal_precision(handle_.get()); +} + +int8_t DataType::decimal_scale() const { + return vx_dtype_decimal_scale(handle_.get()); +} + +namespace { +const vx_struct_fields *struct_fields_or_throw(const vx_dtype *dtype) { + const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); + if (fields == nullptr) { + throw VortexException("dtype is not a struct", ErrorCode::MismatchedTypes); + } + return fields; +} +} // namespace + +std::vector DataType::fields() const { + const std::unique_ptr fields( + struct_fields_or_throw(handle_.get()), + &vx_struct_fields_free); + const uint64_t fields_size = vx_struct_fields_nfields(fields.get()); + std::vector out; + out.reserve(fields_size); + for (uint64_t idx = 0; idx < fields_size; ++idx) { + const vx_view name = vx_struct_fields_field_name(fields.get(), idx); + if (name.ptr == nullptr) { + throw VortexException("error getting field name at index "s + std::to_string(idx), + ErrorCode::Other); + } + const vx_dtype *dtype = vx_struct_fields_field_dtype(fields.get(), idx); + if (dtype == nullptr) { + throw VortexException("error getting dtype at index "s + std::to_string(idx), ErrorCode::Other); + } + out.push_back(Field {{name.ptr, name.len}, DataType(dtype)}); + } + return out; +} + +DataType DataType::list_element() const { + const vx_dtype *element = vx_dtype_list_element(handle_.get()); + if (element == nullptr) { + throw VortexException("dtype is not a list", ErrorCode::MismatchedTypes); + } + return DataType(element); +} + +DataType DataType::fixed_size_list_element() const { + const vx_dtype *element = vx_dtype_fixed_size_list_element(handle_.get()); + if (element == nullptr) { + throw VortexException("dtype is not a fixed-size list", ErrorCode::MismatchedTypes); + } + return DataType(element); +} + +uint32_t DataType::fixed_size_list_size() const { + return vx_dtype_fixed_size_list_size(handle_.get()); +} + +namespace dtype { + +DataType null() { + return Access::adopt(vx_dtype_new_null()); +} +DataType boolean(bool nullable) { + return Access::adopt(vx_dtype_new_bool(nullable)); +} +DataType primitive(PType ptype, bool nullable) { + return Access::adopt(vx_dtype_new_primitive(static_cast(ptype), nullable)); +} +DataType int8(bool nullable) { + return primitive(PType::I8, nullable); +} +DataType int16(bool nullable) { + return primitive(PType::I16, nullable); +} +DataType int32(bool nullable) { + return primitive(PType::I32, nullable); +} +DataType int64(bool nullable) { + return primitive(PType::I64, nullable); +} +DataType uint8(bool nullable) { + return primitive(PType::U8, nullable); +} +DataType uint16(bool nullable) { + return primitive(PType::U16, nullable); +} +DataType uint32(bool nullable) { + return primitive(PType::U32, nullable); +} +DataType uint64(bool nullable) { + return primitive(PType::U64, nullable); +} +DataType float16(bool nullable) { + return primitive(PType::F16, nullable); +} +DataType float32(bool nullable) { + return primitive(PType::F32, nullable); +} +DataType float64(bool nullable) { + return primitive(PType::F64, nullable); +} +DataType utf8(bool nullable) { + return Access::adopt(vx_dtype_new_utf8(nullable)); +} +DataType binary(bool nullable) { + return Access::adopt(vx_dtype_new_binary(nullable)); +} +DataType decimal(uint8_t precision, int8_t scale, bool nullable) { + return Access::adopt(vx_dtype_new_decimal(precision, scale, nullable)); +} +DataType list(DataType element, bool nullable) { + return Access::adopt(vx_dtype_new_list(Access::release(std::move(element)), nullable)); +} +DataType fixed_size_list(DataType element, uint32_t size, bool nullable) { + return Access::adopt( + vx_dtype_new_fixed_size_list(Access::release(std::move(element)), size, nullable)); +} + +DataType struct_(std::initializer_list> fields, bool nullable) { + StructFieldsBuilder builder; + for (const auto &[name, field_dtype] : fields) { + builder.add(name, field_dtype); + } + return std::move(builder).build(nullable); +} + +} // namespace dtype + +void StructFieldsBuilder::Deleter::operator()(vx_struct_fields_builder *ptr) const noexcept { + vx_struct_fields_builder_free(ptr); +} + +StructFieldsBuilder::StructFieldsBuilder() : handle_(vx_struct_fields_builder_new()) { +} + +StructFieldsBuilder &StructFieldsBuilder::add(std::string_view name, const DataType &dtype) & { + vx_error *error = nullptr; + vx_struct_fields_builder_add_field(handle_.get(), + to_view(name), + vx_dtype_clone(Access::c_ptr(dtype)), + &error); + throw_on_error(error); + return *this; +} + +StructFieldsBuilder &&StructFieldsBuilder::add(std::string_view name, const DataType &dtype) && { + add(name, dtype); + return std::move(*this); +} + +DataType StructFieldsBuilder::build(bool nullable) && { + vx_struct_fields *fields = vx_struct_fields_builder_finalize(handle_.release()); + return Access::adopt(vx_dtype_new_struct(fields, nullable)); +} + +} // namespace vortex diff --git a/lang/cpp/src/expression.cpp b/lang/cpp/src/expression.cpp new file mode 100644 index 00000000000..d314ae26293 --- /dev/null +++ b/lang/cpp/src/expression.cpp @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/error.hpp" +#include "vortex/expression.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Expression::Deleter::operator()(const vx_expression *ptr) const noexcept { + vx_expression_free(ptr); +} + +Expression::Expression(const Expression &other) : handle_(vx_expression_clone(other.handle_.get())) { +} + +Expression &Expression::operator=(const Expression &other) { + if (this != &other) { + handle_.reset(vx_expression_clone(other.handle_.get())); + } + return *this; +} + +Expression Expression::operator[](std::string_view field) const { + vx_expression *out = vx_expression_get_item(to_view(field), handle_.get()); + if (out == nullptr) { + throw VortexException("get_item: field name is not valid UTF-8", ErrorCode::InvalidArgument); + } + return Access::adopt(out); +} + +Expression Expression::is_null() const { + return expr::is_null(*this); +} + +template +static Expression select_impl(std::span names, const vx_expression *expr) { + std::vector raw; + raw.reserve(names.size()); + for (const auto &name : names) { + raw.push_back(to_view(name)); + } + return Access::adopt(vx_expression_select(raw.data(), raw.size(), expr)); +} + +Expression Expression::select(std::span names) const { + return select_impl(names, handle_.get()); +} + +Expression Expression::select(std::span names) const { + return select_impl(names, handle_.get()); +} + +Expression Expression::select(std::initializer_list names) const { + std::span span {names.begin(), names.end()}; + return select_impl(span, handle_.get()); +} + +namespace expr { + +Expression root() { + return Access::adopt(vx_expression_root()); +} + +Expression col(std::string_view name) { + return root()[name]; +} + +Expression lit(const Scalar &value) { + vx_error *error = nullptr; + vx_expression *out = vx_expression_literal(Access::c_ptr(value), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Expression binary_op(BinaryOperator op, const Expression &l, const Expression &r) { + return Access::adopt( + vx_expression_binary(static_cast(op), Access::c_ptr(l), Access::c_ptr(r))); +} + +Expression eq(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Eq, l, r); +} +Expression neq(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::NotEq, l, r); +} +Expression lt(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Lt, l, r); +} +Expression lte(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Lte, l, r); +} +Expression gt(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Gt, l, r); +} +Expression gte(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Gte, l, r); +} +Expression add(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Add, l, r); +} +Expression sub(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Sub, l, r); +} +Expression mul(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Mul, l, r); +} +Expression div(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::Div, l, r); +} + +static Expression combine(vx_expression *(*combiner)(const vx_expression *const *, size_t), + std::span children) { + std::vector raw; + raw.reserve(children.size()); + for (const auto &child : children) { + raw.push_back(Access::c_ptr(child)); + } + vx_expression *out = combiner(raw.data(), raw.size()); + if (out == nullptr) { + throw VortexException("empty expression list", ErrorCode::InvalidArgument); + } + return Access::adopt(out); +} + +Expression and_all(std::span children) { + return combine(vx_expression_and, children); +} + +Expression or_all(std::span children) { + return combine(vx_expression_or, children); +} + +Expression logical_not(const Expression &child) { + return Access::adopt(vx_expression_not(Access::c_ptr(child))); +} + +Expression is_null(const Expression &child) { + return Access::adopt(vx_expression_is_null(Access::c_ptr(child))); +} + +Expression list_contains(const Expression &list, const Expression &value) { + return Access::adopt(vx_expression_list_contains(Access::c_ptr(list), Access::c_ptr(value))); +} + +} // namespace expr + +} // namespace vortex diff --git a/lang/cpp/src/f16.cpp b/lang/cpp/src/f16.cpp new file mode 100644 index 00000000000..de1058cc7d4 --- /dev/null +++ b/lang/cpp/src/f16.cpp @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" + +#include +#include + +namespace vortex { + +#if __STDCPP_FLOAT16_T__ != 1 +float16_t::operator float() const { + float result; + const uint32_t sign = (bits >> 15) & 1; + const uint32_t exponent = (bits >> 10) & 0x1F; + const uint32_t mantissa = bits & 0x3FF; + + uint32_t out; + if (exponent == 0x1F) { + out = (sign << 31) | 0x7F800000 | (mantissa << 13); + } else if (exponent == 0) { + if (mantissa == 0) { + out = sign << 31; + } else { + uint32_t m = mantissa; + int e = -1; + do { + m <<= 1; + ++e; + } while ((m & 0x400) == 0); + out = (sign << 31) | ((127 - 15 - e) << 23) | ((m & 0x3FF) << 13); + } + } else { + out = (sign << 31) | ((exponent - 15 + 127) << 23) | (mantissa << 13); + } + std::memcpy(&result, &out, sizeof(result)); + return result; +} +#endif +} // namespace vortex diff --git a/lang/cpp/src/scalar.cpp b/lang/cpp/src/scalar.cpp new file mode 100644 index 00000000000..48a6ce03921 --- /dev/null +++ b/lang/cpp/src/scalar.cpp @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/error.hpp" +#include "vortex/scalar.hpp" + +#include + +#if __STDCPP_FLOAT16_T__ == 1 +#include +#endif +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Scalar::Deleter::operator()(vx_scalar *ptr) const noexcept { + vx_scalar_free(ptr); +} + +Scalar::Scalar(const Scalar &other) : handle_(vx_scalar_clone(other.handle_.get())) { +} + +Scalar &Scalar::operator=(const Scalar &other) { + if (this != &other) { + handle_.reset(vx_scalar_clone(other.handle_.get())); + } + return *this; +} + +bool Scalar::is_null() const { + return vx_scalar_is_null(handle_.get()); +} + +DataType Scalar::dtype() const { + return Access::adopt(vx_scalar_dtype(handle_.get())); +} + +namespace detail { + +Scalar adopt(vx_scalar *raw) { + return Access::adopt(raw); +} + +vx_scalar *make_bool(bool value, bool nullable) { + return vx_scalar_new_bool(value, nullable); +} + +vx_scalar *make_primitive(vx_ptype ptype, const void *value, bool nullable) { + switch (ptype) { + case PTYPE_U8: + return vx_scalar_new_u8(*static_cast(value), nullable); + case PTYPE_U16: + return vx_scalar_new_u16(*static_cast(value), nullable); + case PTYPE_U32: + return vx_scalar_new_u32(*static_cast(value), nullable); + case PTYPE_U64: + return vx_scalar_new_u64(*static_cast(value), nullable); + case PTYPE_I8: + return vx_scalar_new_i8(*static_cast(value), nullable); + case PTYPE_I16: + return vx_scalar_new_i16(*static_cast(value), nullable); + case PTYPE_I32: + return vx_scalar_new_i32(*static_cast(value), nullable); + case PTYPE_I64: + return vx_scalar_new_i64(*static_cast(value), nullable); +#if __STDCPP_FLOAT16_T__ != 1 + case PTYPE_F16: + return vx_scalar_new_f16_bits(static_cast(value)->bits, nullable); +#else + case PTYPE_F16: + return vx_scalar_new_f16_bits(std::bit_cast(*static_cast(value)), + nullable); +#endif + case PTYPE_F32: + return vx_scalar_new_f32(*static_cast(value), nullable); + case PTYPE_F64: + return vx_scalar_new_f64(*static_cast(value), nullable); + } + throw VortexException("unsupported ptype", ErrorCode::InvalidArgument); +} + +vx_scalar *make_utf8(std::string_view value, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_utf8(to_view(value), nullable, &error); + throw_on_error(error); + return out; +} + +vx_scalar *make_binary(BinaryView value, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = + vx_scalar_new_binary(reinterpret_cast(value.data()), value.size(), nullable, &error); + throw_on_error(error); + return out; +} + +} // namespace detail + +namespace scalar { +Scalar null(const DataType &dtype) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_null(Access::c_ptr(dtype), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_decimal_i32(value, precision, scale, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_decimal_i64(value, precision, scale, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +} // namespace scalar + +} // namespace vortex diff --git a/lang/cpp/src/scan.cpp b/lang/cpp/src/scan.cpp new file mode 100644 index 00000000000..8c7967b010d --- /dev/null +++ b/lang/cpp/src/scan.cpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/scan.hpp" + +#include + +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; + +ArrowStream::ArrowStream(ArrowStream &&other) noexcept + : session_(std::move(other.session_)), stream_(other.stream_) { + other.stream_ = ArrowArrayStream {}; +} + +ArrowStream &ArrowStream::operator=(ArrowStream &&other) noexcept { + if (this != &other) { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } + session_ = std::move(other.session_); + stream_ = other.stream_; + other.stream_ = ArrowArrayStream {}; + } + return *this; +} + +ArrowStream::~ArrowStream() { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } +} + +void Partition::Deleter::operator()(vx_partition *ptr) const noexcept { + vx_partition_free(ptr); +} + +Estimate Partition::row_count() const { + vx_estimate raw {}; + vx_error *error = nullptr; + vx_partition_row_count(handle_.get(), &raw, &error); + throw_on_error(error); + return Estimate(raw); +} + +std::optional Partition::next() { + vx_error *error = nullptr; + const vx_array *array = vx_partition_next(handle_.get(), &error); + throw_on_error(error); + if (array == nullptr) { + return std::nullopt; + } + return Access::adopt(array); +} + +ArrowStream Partition::into_arrow_stream() && { + ArrowArrayStream stream {}; + vx_error *error = nullptr; + // Consumes handle even on error + vx_partition_scan_arrow(Access::c_ptr(session_), handle_.release(), &stream, &error); + throw_on_error(error); + return Access::adopt(std::move(session_), stream); +} + +void Scan::Deleter::operator()(vx_scan *ptr) const noexcept { + vx_scan_free(ptr); +} + +DataType Scan::dtype() const { + vx_error *error = nullptr; + const vx_dtype *out = vx_scan_dtype(handle_.get(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +std::optional Scan::next_partition() { + vx_partition *partition = nullptr; + { + const std::lock_guard guard(*mutex_); + vx_error *error = nullptr; + partition = vx_scan_next_partition(handle_.get(), &error); + throw_on_error(error); + } + if (partition == nullptr) { + return std::nullopt; + } + return Access::adopt(partition, session_); +} +} // namespace vortex diff --git a/lang/cpp/src/session.cpp b/lang/cpp/src/session.cpp new file mode 100644 index 00000000000..fb2753476bd --- /dev/null +++ b/lang/cpp/src/session.cpp @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/session.hpp" + +#include + +namespace vortex { + +void Session::Deleter::operator()(vx_session *ptr) const noexcept { + vx_session_free(ptr); +} + +Session::Session() : handle_(vx_session_new()) { +} + +Session::Session(const Session &other) : handle_(vx_session_clone(other.handle_.get())) { +} + +Session &Session::operator=(const Session &other) { + if (this != &other) { + handle_.reset(vx_session_clone(other.handle_.get())); + } + return *this; +} + +} // namespace vortex diff --git a/lang/cpp/src/writer.cpp b/lang/cpp/src/writer.cpp new file mode 100644 index 00000000000..4adc5d77e50 --- /dev/null +++ b/lang/cpp/src/writer.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/writer.hpp" +#include "vortex/session.hpp" + +#include + +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Writer::Deleter::operator()(vx_array_sink *ptr) const noexcept { + vx_array_sink_abort(ptr); +} + +Writer Writer::open(const Session &session, std::string_view path, const DataType &dtype) { + vx_error *error = nullptr; + vx_array_sink *sink = + vx_array_sink_open_file(Access::c_ptr(session), to_view(path), Access::c_ptr(dtype), &error); + throw_on_error(error); + return Writer(sink); +} + +void Writer::push(const Array &array) { + if (handle_ == nullptr) { + throw VortexException("null handle_", ErrorCode::InvalidArgument); + } + vx_error *error = nullptr; + vx_array_sink_push(handle_.get(), Access::c_ptr(array), &error); + throw_on_error(error); +} + +void Writer::finish() { + if (handle_ == nullptr) { + throw VortexException("finish() called twice", ErrorCode::InvalidArgument); + } + vx_error *error = nullptr; + vx_array_sink_close(handle_.release(), &error); + throw_on_error(error); +} +} // namespace vortex From 5ed15ec5671294dad8986f6d36a7297605270717 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 13:31:04 +0100 Subject: [PATCH 2/6] review fix Signed-off-by: Mikhail Kot --- docs/api/cpp/index.rst | 2 +- lang/cpp/include/vortex/array.hpp | 20 +++++++++----------- lang/cpp/include/vortex/data_source.hpp | 5 +++-- lang/cpp/include/vortex/scan.hpp | 20 +++++--------------- lang/cpp/src/array.cpp | 11 ++++------- lang/cpp/src/data_source.cpp | 2 +- vortex-ffi/src/array.rs | 7 ++++++- vortex-ffi/src/sink.rs | 2 +- 8 files changed, 30 insertions(+), 39 deletions(-) diff --git a/docs/api/cpp/index.rst b/docs/api/cpp/index.rst index ea1c415ac93..6d1f33b177d 100644 --- a/docs/api/cpp/index.rst +++ b/docs/api/cpp/index.rst @@ -187,7 +187,7 @@ Source code for this example is `writer.cpp Expression age_gt_10 = expr::gt(expr::col("age"), expr::lit(10)); Array validity_array = array.apply(age_gt_10); - const Validity validity {ValidityType::Array, validity_array}; + const Validity validity = Validity::from_array(validity_array); Array array2 = make_struct({ {"age", age}, {"height", Array::primitive(height_buffer, validity)}, diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp index ad06b1eb7d5..1b78e31536f 100644 --- a/lang/cpp/include/vortex/array.hpp +++ b/lang/cpp/include/vortex/array.hpp @@ -30,6 +30,9 @@ class Array; class StringView; class BytesView; +/* + * Validity type tells us whether there are null/invalid values in an Array. + */ enum class ValidityType { // Items can't be null NonNullable = VX_VALIDITY_NON_NULLABLE, @@ -41,10 +44,6 @@ enum class ValidityType { Array = VX_VALIDITY_ARRAY, }; -constexpr ValidityType NonNullable = ValidityType::NonNullable; -constexpr ValidityType AllValid = ValidityType::AllValid; -constexpr ValidityType AllInvalid = ValidityType::AllInvalid; - /** * Array per-element validity of type ValidityType. * If ValidityType is ValidityType::Array, holds a boolean array @@ -58,7 +57,8 @@ class Validity { // NonNullable/AllValid/AllInvalid constructor // NOLINTNEXTLINE(google-explicit-constructor) Validity(ValidityType type); - Validity(ValidityType type, const Array &array); + // Validity determined by a boolean array, true = valid, false = invalid. + static Validity from_array(const Array &bools); Validity(const Validity &other); Validity(Validity &&other) noexcept; @@ -83,9 +83,6 @@ class Validity { const vx_array *array_; }; -// Validity determined by a boolean array, true = valid, false = invalid. -Validity ValidityArray(const Array &bools); - namespace detail { // Validity bitmap for typed views. Owns the arrays that back the bits class ValidityBits { @@ -130,7 +127,7 @@ class Array { * auto array = Array::primitive(buffer); */ template - static Array primitive(std::span data, const Validity &validity = NonNullable) { + static Array primitive(std::span data, const Validity &validity = ValidityType::NonNullable) { return primitive_raw(detail::to_ptype(), data.data(), data.size(), validity); } @@ -241,6 +238,7 @@ class Array { * * Example: * + * using enum ValidityType; * std::array age_buffer = {0, 1, 2}; * std::array height_buffer = {0, 1, 2}; * Array ages = Array::primitive(age_buffer); @@ -250,7 +248,7 @@ class Array { * NonNullable); */ Array make_struct(std::initializer_list> fields, - const Validity &validity = NonNullable); + const Validity &validity = ValidityType::NonNullable); /** * Create a Struct Array from a dynamic number of fields. @@ -387,7 +385,7 @@ class StringView { }; /** - * Read-only view over a Binary array. + * Read-only view over a Bytes array. * * Byte spans borrow from the view's canonical copy and are valid as long as * the view lives. diff --git a/lang/cpp/include/vortex/data_source.hpp b/lang/cpp/include/vortex/data_source.hpp index af99090601f..f3221f37068 100644 --- a/lang/cpp/include/vortex/data_source.hpp +++ b/lang/cpp/include/vortex/data_source.hpp @@ -32,7 +32,8 @@ class DataSource { /** * Create a DataSource from an in-memory Vortex file. Borrows the buffer: - * caller must keep it alive and unmodified while DataSource is alive. + * caller must keep it alive and unmodified while DataSource, Scans, + * Partitions, and Arrays obtained from this buffer are alive. */ static DataSource from_buffer(const Session &session, std::span data); @@ -43,7 +44,7 @@ class DataSource { Estimate row_count() const; DataType dtype() const; - Scan scan(ScanOptions options = {}) const; + Scan scan(const ScanOptions &options = {}) const; private: friend struct detail::Access; diff --git a/lang/cpp/include/vortex/scan.hpp b/lang/cpp/include/vortex/scan.hpp index 9113ce1d00a..b8bee5ea141 100644 --- a/lang/cpp/include/vortex/scan.hpp +++ b/lang/cpp/include/vortex/scan.hpp @@ -21,7 +21,7 @@ namespace vortex { namespace detail { // range-for support for Scan and Partition -template +template class PullRange { public: class iterator { @@ -37,7 +37,7 @@ class PullRange { return *cur_; } iterator &operator++() { - cur_ = src_->pull(); + cur_ = Next(src_); return *this; } void operator++(int) { @@ -55,7 +55,7 @@ class PullRange { explicit PullRange(Source &src) : src_(&src) { } iterator begin() { - return iterator(src_, src_->pull()); + return iterator(src_, Next(src_)); } std::default_sentinel_t end() { return std::default_sentinel; @@ -111,7 +111,7 @@ class Partition { // range-for over Arrays auto batches() & { - return detail::PullRange(*this); + return detail::PullRange(*this); } auto batches() && = delete; @@ -123,14 +123,9 @@ class Partition { private: friend struct detail::Access; - friend class detail::PullRange; Partition(vx_partition *owned, Session session) : handle_(owned), session_(std::move(session)) { } - std::optional pull() { - return next(); - } - struct Deleter { void operator()(vx_partition *ptr) const noexcept; }; @@ -221,22 +216,17 @@ class Scan { // range-for over partitions auto partitions() & { - return detail::PullRange(*this); + return detail::PullRange(*this); } auto partitions() && = delete; private: friend struct detail::Access; - friend class detail::PullRange; Scan(vx_scan *owned, Estimate estimate, Session session) : handle_(owned), mutex_(std::make_unique()), estimate_(estimate), session_(std::move(session)) { } - std::optional pull() { - return next_partition(); - } - struct Deleter { void operator()(vx_scan *ptr) const noexcept; }; diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp index b0177319857..d312a83037b 100644 --- a/lang/cpp/src/array.cpp +++ b/lang/cpp/src/array.cpp @@ -29,8 +29,8 @@ Validity::Validity(ValidityType type) : type_(type), array_(nullptr) { } } -Validity::Validity(ValidityType type, const Array &array) - : type_(type), array_(vx_array_clone(Access::c_ptr(array))) { +Validity Validity::from_array(const Array &bools) { + return {ValidityType::Array, vx_array_clone(Access::c_ptr(bools))}; } Validity::Validity(Validity &&other) noexcept : type_(other.type_), array_(other.array_) { @@ -67,11 +67,9 @@ Array Validity::array() const { return Access::adopt(vx_array_clone(array_)); } -Validity ValidityArray(const Array &bools) { - return Validity(ValidityType::Array, vx_array_clone(Access::c_ptr(bools))); -} +namespace detail { -bool detail::ValidityBits::is_null(size_t index) const noexcept { +bool ValidityBits::is_null(size_t index) const noexcept { if (all_invalid_) { return true; } @@ -82,7 +80,6 @@ bool detail::ValidityBits::is_null(size_t index) const noexcept { return (bits_[bit / 8] >> (bit % 8) & 1) == 0; } -namespace detail { ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) { vx_validity raw {}; vx_error *error = nullptr; diff --git a/lang/cpp/src/data_source.cpp b/lang/cpp/src/data_source.cpp index f032f47128d..6aa8d3e3107 100644 --- a/lang/cpp/src/data_source.cpp +++ b/lang/cpp/src/data_source.cpp @@ -84,7 +84,7 @@ DataType DataSource::dtype() const { return Access::adopt(vx_data_source_dtype(handle_.get())); } -Scan DataSource::scan(ScanOptions options) const { +Scan DataSource::scan(const ScanOptions &options) const { vx_scan_options raw {}; raw.projection = options.projection.has_value() ? Access::c_ptr(*options.projection) : nullptr; raw.filter = options.filter.has_value() ? Access::c_ptr(*options.filter) : nullptr; diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 3eda7732190..577bd4d8be5 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -5,6 +5,7 @@ //! FFI interface for working with Vortex Arrays. use std::ffi::c_void; use std::ptr; +use std::ptr::NonNull; use std::sync::Arc; use arrow_array::array::make_array; @@ -305,7 +306,11 @@ unsafe fn primitive_from_raw( len: usize, validity: &vx_validity, ) -> *const vx_array { - let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; + let slice = if ptr.is_null() { + unsafe { std::slice::from_raw_parts(NonNull::dangling().as_ptr(), len) } + } else { + unsafe { std::slice::from_raw_parts(ptr, len) } + }; let buffer = Buffer::copy_from(slice); let array = PrimitiveArray::new(buffer, validity.into()); vx_array::new(Arc::new(array.into_array())) diff --git a/vortex-ffi/src/sink.rs b/vortex-ffi/src/sink.rs index 7ca149d0731..f307cfc8bc0 100644 --- a/vortex-ffi/src/sink.rs +++ b/vortex-ffi/src/sink.rs @@ -65,7 +65,7 @@ pub unsafe extern "C-unwind" fn vx_array_sink_open_file( error_out: *mut *mut vx_error, ) -> *mut vx_array_sink { try_or_default(error_out, || { - let session = vx_session::as_ref(session); + let session = vx_session::as_ref(session).clone(); if path.ptr.is_null() { vortex_bail!("null path"); From c2bd5462a8c51d8edde3c198b8c7cf92ba2e122e Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 14:31:46 +0100 Subject: [PATCH 3/6] fix Signed-off-by: Mikhail Kot --- lang/cpp/include/vortex/array.hpp | 24 ++++++++++++------------ lang/cpp/include/vortex/common.hpp | 4 ++-- lang/cpp/include/vortex/dtype.hpp | 2 +- lang/cpp/include/vortex/error.hpp | 4 ++-- lang/cpp/include/vortex/estimate.hpp | 4 ++-- lang/cpp/include/vortex/expression.hpp | 2 +- lang/cpp/include/vortex/scalar.hpp | 2 +- lang/cpp/include/vortex/scan.hpp | 4 ++-- lang/cpp/src/array.cpp | 2 +- lang/cpp/src/scalar.cpp | 1 - 10 files changed, 24 insertions(+), 25 deletions(-) diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp index 1b78e31536f..1891d45a1ee 100644 --- a/lang/cpp/include/vortex/array.hpp +++ b/lang/cpp/include/vortex/array.hpp @@ -66,7 +66,7 @@ class Validity { Validity &operator=(Validity &&other) noexcept; ~Validity(); - ValidityType type() const noexcept { + ValidityType type() const { return type_; } @@ -93,7 +93,7 @@ class ValidityBits { ValidityBits &operator=(const ValidityBits &) = delete; ~ValidityBits(); - bool is_null(size_t index) const noexcept; + bool is_null(size_t index) const; private: friend class vortex::Array; @@ -220,7 +220,7 @@ class Array { explicit Array(const vx_array *owned) : handle_(owned) { } - const vx_array *release() && noexcept { + const vx_array *release() && { return handle_.release(); } @@ -301,13 +301,13 @@ class PrimitiveView { * Get raw values from this view. Values at null/invalid positions are * unspecified. */ - std::span values() const noexcept { + std::span values() const { return {data_, size_}; } - bool is_null(size_t index) const noexcept { + bool is_null(size_t index) const { return validity_.is_null(index); } - size_t size() const noexcept { + size_t size() const { return size_; } @@ -335,10 +335,10 @@ class PrimitiveView { * unspecified. */ bool value(size_t index) const; - bool is_null(size_t index) const noexcept { + bool is_null(size_t index) const { return validity_.is_null(index); } - size_t size() const noexcept { + size_t size() const { return size_; } @@ -366,10 +366,10 @@ class StringView { * unspecified. */ std::string_view operator[](size_t index) const; - bool is_null(size_t index) const noexcept { + bool is_null(size_t index) const { return validity_.is_null(index); } - size_t size() const noexcept { + size_t size() const { return size_; } @@ -397,10 +397,10 @@ class BytesView { * unspecified. */ BinaryView operator[](size_t index) const; - bool is_null(size_t index) const noexcept { + bool is_null(size_t index) const { return validity_.is_null(index); } - size_t size() const noexcept { + size_t size() const { return size_; } diff --git a/lang/cpp/include/vortex/common.hpp b/lang/cpp/include/vortex/common.hpp index cf70888f7ce..1248f6d53bd 100644 --- a/lang/cpp/include/vortex/common.hpp +++ b/lang/cpp/include/vortex/common.hpp @@ -37,11 +37,11 @@ struct Access { return T(std::forward(args)...); } template - static auto release(T &&t) noexcept { + static auto release(T &&t) { return std::forward(t).release(); } template - static auto c_ptr(const T &t) noexcept { + static auto c_ptr(const T &t) { return t.handle_.get(); } }; diff --git a/lang/cpp/include/vortex/dtype.hpp b/lang/cpp/include/vortex/dtype.hpp index edb62dd839d..bd915355470 100644 --- a/lang/cpp/include/vortex/dtype.hpp +++ b/lang/cpp/include/vortex/dtype.hpp @@ -99,7 +99,7 @@ class DataType { private: friend struct detail::Access; explicit DataType(const vx_dtype *owned); - const vx_dtype *release() && noexcept { + const vx_dtype *release() && { return handle_.release(); } diff --git a/lang/cpp/include/vortex/error.hpp b/lang/cpp/include/vortex/error.hpp index ba76b5c9385..8793be76dbb 100644 --- a/lang/cpp/include/vortex/error.hpp +++ b/lang/cpp/include/vortex/error.hpp @@ -27,7 +27,7 @@ class VortexException : public std::runtime_error { VortexException(const std::string &message, ErrorCode code) : std::runtime_error(message), code_(code) { } - ErrorCode code() const noexcept { + ErrorCode code() const { return code_; } @@ -48,7 +48,7 @@ inline void throw_on_error(vx_error *error) { throw VortexException(message, code); } -inline vx_view to_view(std::string_view view) noexcept { +inline vx_view to_view(std::string_view view) { return {view.data(), view.size()}; } } // namespace detail diff --git a/lang/cpp/include/vortex/estimate.hpp b/lang/cpp/include/vortex/estimate.hpp index 19fc174bf5c..1b412555b0f 100644 --- a/lang/cpp/include/vortex/estimate.hpp +++ b/lang/cpp/include/vortex/estimate.hpp @@ -21,7 +21,7 @@ class Estimate { explicit Estimate(vx_estimate raw) : raw_(raw) { } - inline EstimateType type() const noexcept { + inline EstimateType type() const { return static_cast(raw_.type); } @@ -36,7 +36,7 @@ class Estimate { return raw_.estimate; } - inline uint64_t value_or(uint64_t fallback) const noexcept { + inline uint64_t value_or(uint64_t fallback) const { return type() == EstimateType::Unknown ? fallback : raw_.estimate; } diff --git a/lang/cpp/include/vortex/expression.hpp b/lang/cpp/include/vortex/expression.hpp index 86486557e21..01547450ae5 100644 --- a/lang/cpp/include/vortex/expression.hpp +++ b/lang/cpp/include/vortex/expression.hpp @@ -50,7 +50,7 @@ class Expression { friend struct detail::Access; explicit Expression(const vx_expression *owned) : handle_(owned) { } - const vx_expression *release() && noexcept { + const vx_expression *release() && { return handle_.release(); } diff --git a/lang/cpp/include/vortex/scalar.hpp b/lang/cpp/include/vortex/scalar.hpp index 02dc006b93b..47d94408eed 100644 --- a/lang/cpp/include/vortex/scalar.hpp +++ b/lang/cpp/include/vortex/scalar.hpp @@ -29,7 +29,7 @@ class Scalar { friend struct detail::Access; explicit Scalar(vx_scalar *owned) : handle_(owned) { } - vx_scalar *release() && noexcept { + vx_scalar *release() && { return handle_.release(); } diff --git a/lang/cpp/include/vortex/scan.hpp b/lang/cpp/include/vortex/scan.hpp index b8bee5ea141..7175f494bef 100644 --- a/lang/cpp/include/vortex/scan.hpp +++ b/lang/cpp/include/vortex/scan.hpp @@ -75,7 +75,7 @@ class ArrowStream { ArrowStream &operator=(ArrowStream &&other) noexcept; ~ArrowStream(); - ArrowArrayStream *raw() noexcept { + ArrowArrayStream *raw() { return &stream_; } @@ -199,7 +199,7 @@ class Scan { Scan(Scan &&) noexcept = default; Scan &operator=(Scan &&) noexcept = default; - Estimate partition_count() const noexcept { + Estimate partition_count() const { return estimate_; } diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp index d312a83037b..7ae1e7e1434 100644 --- a/lang/cpp/src/array.cpp +++ b/lang/cpp/src/array.cpp @@ -69,7 +69,7 @@ Array Validity::array() const { namespace detail { -bool ValidityBits::is_null(size_t index) const noexcept { +bool ValidityBits::is_null(size_t index) const { if (all_invalid_) { return true; } diff --git a/lang/cpp/src/scalar.cpp b/lang/cpp/src/scalar.cpp index 48a6ce03921..ee64ab88f60 100644 --- a/lang/cpp/src/scalar.cpp +++ b/lang/cpp/src/scalar.cpp @@ -16,7 +16,6 @@ namespace vortex { using detail::Access; using detail::throw_on_error; -using detail::to_view; void Scalar::Deleter::operator()(vx_scalar *ptr) const noexcept { vx_scalar_free(ptr); From a17a71e64ea43f96ff43bac31f313858fbde8b58 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 14:53:53 +0100 Subject: [PATCH 4/6] decimal Signed-off-by: Mikhail Kot --- lang/cpp/include/vortex/scalar.hpp | 23 ++++++++++++++++++++--- lang/cpp/src/scalar.cpp | 16 ---------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lang/cpp/include/vortex/scalar.hpp b/lang/cpp/include/vortex/scalar.hpp index 47d94408eed..f454d918df3 100644 --- a/lang/cpp/include/vortex/scalar.hpp +++ b/lang/cpp/include/vortex/scalar.hpp @@ -4,6 +4,7 @@ #include "vortex/common.hpp" #include "vortex/dtype.hpp" +#include "vortex/error.hpp" #include @@ -67,10 +68,26 @@ Scalar of(T value, bool nullable = false) { } } +template +Scalar decimal(T value, uint8_t precision, int8_t scale, bool nullable = false) { + vx_error *error = nullptr; + vx_scalar *out = nullptr; + if constexpr (std::is_same_v) { + out = vx_scalar_new_decimal_i8(value, precision, scale, nullable, &error); + } else if constexpr (std::is_same_v) { + out = vx_scalar_new_decimal_i16(value, precision, scale, nullable, &error); + } else if constexpr (std::is_same_v) { + out = vx_scalar_new_decimal_i32(value, precision, scale, nullable, &error); + } else if constexpr (std::is_same_v) { + out = vx_scalar_new_decimal_i64(value, precision, scale, nullable, &error); + } else { + static_assert(false, "can't construct decimal of following scale"); + } + detail::throw_on_error(error); + return detail::Access::adopt(out); +} + // A typed null of (a nullable copy of) a given DataType. Scalar null(const DataType &dtype); - -Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable = false); -Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable = false); } // namespace scalar } // namespace vortex diff --git a/lang/cpp/src/scalar.cpp b/lang/cpp/src/scalar.cpp index ee64ab88f60..2bef2699bba 100644 --- a/lang/cpp/src/scalar.cpp +++ b/lang/cpp/src/scalar.cpp @@ -107,21 +107,5 @@ Scalar null(const DataType &dtype) { throw_on_error(error); return Access::adopt(out); } - -Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable) { - vx_error *error = nullptr; - vx_scalar *out = vx_scalar_new_decimal_i32(value, precision, scale, nullable, &error); - throw_on_error(error); - return Access::adopt(out); -} - -Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable) { - vx_error *error = nullptr; - vx_scalar *out = vx_scalar_new_decimal_i64(value, precision, scale, nullable, &error); - throw_on_error(error); - return Access::adopt(out); -} - } // namespace scalar - } // namespace vortex From 49e4b3f71deadb76447045eea157e26c26f787dd Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 15:10:49 +0100 Subject: [PATCH 5/6] column fields for make_struct --- lang/cpp/include/vortex/array.hpp | 47 ++++++------------------------- lang/cpp/src/array.cpp | 46 +++++++++++------------------- 2 files changed, 24 insertions(+), 69 deletions(-) diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp index 1891d45a1ee..82c1b5d747d 100644 --- a/lang/cpp/include/vortex/array.hpp +++ b/lang/cpp/include/vortex/array.hpp @@ -233,6 +233,12 @@ class Array { std::unique_ptr handle_; }; +// Column field of a Struct Array +struct ColumnField { + std::string name; + Array column; +}; + /** * Create a Struct array from named columns of equal length. * @@ -247,47 +253,10 @@ class Array { * {{"age", ages}, {"height", heights}}, * NonNullable); */ -Array make_struct(std::initializer_list> fields, +Array make_struct(std::span fields, const Validity &validity = ValidityType::NonNullable); +Array make_struct(std::initializer_list fields, const Validity &validity = ValidityType::NonNullable); -/** - * Create a Struct Array from a dynamic number of fields. - * Prefer make_struct() if you know all fields beforehand. - * - * Example: - * - * std::array age_buffer = {0, 1, 2}; - * std::array height_buffer = {0, 1, 2}; - * Array ages = Array::primitive(age_buffer); - * Array heights = Array::primitive(height_buffer); - * - * StructArrayBuilder b(NonNullable, 2); - * Array result = builder - * .add("age", ages) - * .add("height", heights) - * .build(); - */ -class StructArrayBuilder { -public: - explicit StructArrayBuilder(const Validity &validity, size_t capacity = 0); - StructArrayBuilder(const StructArrayBuilder &) = delete; - StructArrayBuilder &operator=(const StructArrayBuilder &) = delete; - StructArrayBuilder(StructArrayBuilder &&) noexcept = default; - StructArrayBuilder &operator=(StructArrayBuilder &&) noexcept = default; - - StructArrayBuilder &add(std::string_view name, const Array &field) &; - StructArrayBuilder &&add(std::string_view name, const Array &field) &&; - - // Consume the builder and return and Array - Array build() &&; - -private: - struct Deleter { - void operator()(vx_struct_column_builder *ptr) const noexcept; - }; - std::unique_ptr handle_; -}; - /** * Typed read-only view over a Primitive array. * diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp index 7ae1e7e1434..11b012449e2 100644 --- a/lang/cpp/src/array.cpp +++ b/lang/cpp/src/array.cpp @@ -259,49 +259,35 @@ Array Array::canonicalize(const Session &session) const { return Access::adopt(out); } -Array make_struct(std::initializer_list> fields, - const Validity &validity) { - StructArrayBuilder builder(validity, fields.size()); - for (const auto &[name, field] : fields) { - builder.add(name, field); - } - return std::move(builder).build(); -} - -void StructArrayBuilder::Deleter::operator()(vx_struct_column_builder *ptr) const noexcept { - vx_struct_column_builder_free(ptr); -} - -StructArrayBuilder::StructArrayBuilder(const Validity &validity, size_t capacity) { - std::optional keep_alive; +Array make_struct(std::span fields, const Validity &validity) { vx_validity raw {}; raw.type = static_cast(validity.type()); + + std::optional keep_alive; if (validity.type() == ValidityType::Array) { keep_alive = validity.array(); raw.array = Access::c_ptr(*keep_alive); } - handle_.reset(vx_struct_column_builder_new(&raw, capacity)); -} -StructArrayBuilder &StructArrayBuilder::add(std::string_view name, const Array &field) & { - vx_error *error = nullptr; - vx_struct_column_builder_add_field(handle_.get(), detail::to_view(name), Access::c_ptr(field), &error); - throw_on_error(error); - return *this; -} - -StructArrayBuilder &&StructArrayBuilder::add(std::string_view name, const Array &field) && { - add(name, field); - return std::move(*this); -} + std::unique_ptr handle( + vx_struct_column_builder_new(&raw, fields.size()), + &vx_struct_column_builder_free); -Array StructArrayBuilder::build() && { vx_error *error = nullptr; - const vx_array *out = vx_struct_column_builder_finalize(handle_.release(), &error); + for (const auto &[name, array] : fields) { + vx_struct_column_builder_add_field(handle.get(), detail::to_view(name), Access::c_ptr(array), &error); + throw_on_error(error); + } + + const vx_array *out = vx_struct_column_builder_finalize(handle.release(), &error); throw_on_error(error); return Access::adopt(out); } +Array make_struct(std::initializer_list fields, const Validity &validity) { + return make_struct({fields.begin(), fields.end()}, validity); +} + PrimitiveView Array::bools(const Session &session) const { Array canonical = canonicalize(session); const vx_array *raw = Access::c_ptr(canonical); From b4aec09919568ea5f2a94c760ad65f774e5f6752 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 15:22:01 +0100 Subject: [PATCH 6/6] fix Signed-off-by: Mikhail Kot --- lang/cpp/CMakeLists.txt | 12 +++++++ lang/cpp/include/vortex/dtype.hpp | 45 ++++--------------------- lang/cpp/src/dtype.cpp | 55 +++++++++++-------------------- 3 files changed, 38 insertions(+), 74 deletions(-) diff --git a/lang/cpp/CMakeLists.txt b/lang/cpp/CMakeLists.txt index c2a792080a0..28099a44baa 100644 --- a/lang/cpp/CMakeLists.txt +++ b/lang/cpp/CMakeLists.txt @@ -98,6 +98,13 @@ target_include_directories(vortex_cxx PUBLIC ) target_link_libraries(vortex_cxx PUBLIC vortex_ffi) add_library(vortex::cxx ALIAS vortex_cxx) +target_compile_features(vortex_cxx PUBLIC cxx_std_20) + +set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") + +if(APPLE) + target_link_libraries(vortex_cxx PRIVATE ${APPLE_LINK_FLAGS}) +endif() if(TARGET vortex_ffi_shared) add_library(vortex_cxx_shared SHARED ${VORTEX_CXX_SOURCES}) @@ -109,4 +116,9 @@ if(TARGET vortex_ffi_shared) ) target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared) add_library(vortex::cxx_shared ALIAS vortex_cxx_shared) + target_compile_features(vortex_cxx_shared PUBLIC cxx_std_20) + + if(APPLE) + target_link_libraries(vortex_cxx_shared PRIVATE ${APPLE_LINK_FLAGS}) + endif() endif() diff --git a/lang/cpp/include/vortex/dtype.hpp b/lang/cpp/include/vortex/dtype.hpp index bd915355470..1be5007c6d6 100644 --- a/lang/cpp/include/vortex/dtype.hpp +++ b/lang/cpp/include/vortex/dtype.hpp @@ -16,7 +16,7 @@ namespace vortex { -struct Field; +struct StructField; enum class DataTypeVariant { Null = DTYPE_NULL, @@ -88,7 +88,7 @@ class DataType { * For a Struct dtype, return its fields in order. * Throws if DataType is not Struct. */ - std::vector fields() const; + std::vector fields() const; // List accessors. Valid only on List and FixedSizeList dtypes @@ -109,8 +109,8 @@ class DataType { std::unique_ptr handle_; }; -// Field of a Struct dtype -struct Field { +// Field of a Struct DataType. +struct StructField { std::string name; DataType dtype; }; @@ -150,40 +150,7 @@ DataType fixed_size_list(DataType element, uint32_t size, bool nullable = false) * {"height", dtype::uint16(Nullable)}} * ); */ -DataType struct_(std::initializer_list> fields, bool nullable = false); +DataType struct_(std::span fields, bool nullable = false); +DataType struct_(std::initializer_list fields, bool nullable = false); } // namespace dtype - -/* - * Create a DataTypeVariant::Struct dynamically. - * Prefer dtype::struct_ if fields are known beforehand. - * - * Example: - * - * using dtype::Nullable; - * DataType dtype = StructFieldsBuilder() - * .add("age", dtype::uint8()) - * .add("height", dtype::uint16(Nullable)}) - * .build(Nullable); - * ); - */ -class StructFieldsBuilder { -public: - StructFieldsBuilder(); - StructFieldsBuilder(const StructFieldsBuilder &) = delete; - StructFieldsBuilder &operator=(const StructFieldsBuilder &) = delete; - StructFieldsBuilder(StructFieldsBuilder &&) noexcept = default; - StructFieldsBuilder &operator=(StructFieldsBuilder &&) noexcept = default; - - StructFieldsBuilder &add(std::string_view name, const DataType &dtype) &; - StructFieldsBuilder &&add(std::string_view name, const DataType &dtype) &&; - - // Consume the builder and return a DataType - DataType build(bool nullable = false) &&; - -private: - struct Deleter { - void operator()(vx_struct_fields_builder *ptr) const noexcept; - }; - std::unique_ptr handle_; -}; } // namespace vortex diff --git a/lang/cpp/src/dtype.cpp b/lang/cpp/src/dtype.cpp index c5f5c7947bc..808ead6fd50 100644 --- a/lang/cpp/src/dtype.cpp +++ b/lang/cpp/src/dtype.cpp @@ -82,12 +82,12 @@ const vx_struct_fields *struct_fields_or_throw(const vx_dtype *dtype) { } } // namespace -std::vector DataType::fields() const { +std::vector DataType::fields() const { const std::unique_ptr fields( struct_fields_or_throw(handle_.get()), &vx_struct_fields_free); const uint64_t fields_size = vx_struct_fields_nfields(fields.get()); - std::vector out; + std::vector out; out.reserve(fields_size); for (uint64_t idx = 0; idx < fields_size; ++idx) { const vx_view name = vx_struct_fields_field_name(fields.get(), idx); @@ -99,7 +99,7 @@ std::vector DataType::fields() const { if (dtype == nullptr) { throw VortexException("error getting dtype at index "s + std::to_string(idx), ErrorCode::Other); } - out.push_back(Field {{name.ptr, name.len}, DataType(dtype)}); + out.push_back(StructField {{name.ptr, name.len}, DataType(dtype)}); } return out; } @@ -185,41 +185,26 @@ DataType fixed_size_list(DataType element, uint32_t size, bool nullable) { vx_dtype_new_fixed_size_list(Access::release(std::move(element)), size, nullable)); } -DataType struct_(std::initializer_list> fields, bool nullable) { - StructFieldsBuilder builder; - for (const auto &[name, field_dtype] : fields) { - builder.add(name, field_dtype); - } - return std::move(builder).build(nullable); -} - -} // namespace dtype - -void StructFieldsBuilder::Deleter::operator()(vx_struct_fields_builder *ptr) const noexcept { - vx_struct_fields_builder_free(ptr); -} - -StructFieldsBuilder::StructFieldsBuilder() : handle_(vx_struct_fields_builder_new()) { -} - -StructFieldsBuilder &StructFieldsBuilder::add(std::string_view name, const DataType &dtype) & { +DataType struct_(std::span fields, bool nullable) { vx_error *error = nullptr; - vx_struct_fields_builder_add_field(handle_.get(), - to_view(name), - vx_dtype_clone(Access::c_ptr(dtype)), - &error); - throw_on_error(error); - return *this; -} - -StructFieldsBuilder &&StructFieldsBuilder::add(std::string_view name, const DataType &dtype) && { - add(name, dtype); - return std::move(*this); + std::unique_ptr handle( + vx_struct_fields_builder_new(), + vx_struct_fields_builder_free); + + for (const auto &[name, dtype] : fields) { + vx_struct_fields_builder_add_field(handle.get(), + to_view(name), + vx_dtype_clone(Access::c_ptr(dtype)), + &error); + throw_on_error(error); + } + vx_struct_fields *ffi_fields = vx_struct_fields_builder_finalize(handle.release()); + return Access::adopt(vx_dtype_new_struct(ffi_fields, nullable)); } -DataType StructFieldsBuilder::build(bool nullable) && { - vx_struct_fields *fields = vx_struct_fields_builder_finalize(handle_.release()); - return Access::adopt(vx_dtype_new_struct(fields, nullable)); +DataType struct_(std::initializer_list fields, bool nullable) { + return struct_({fields.begin(), fields.end()}, nullable); } +} // namespace dtype } // namespace vortex