duckdb · Mytherin · Feb 4, 2025 · Jan 27, 2025 · Jan 28, 2025 · Jan 28, 2025
diff --git a/.github/actions/build_extensions/action.yml b/.github/actions/build_extensions/action.yml
@@ -233,7 +233,7 @@ runs:
 
     # Run the unittests (excluding the out-of-tree tests) with the extensions that we deployed to S3
     - name: Test deployed extensions
-      if: ${{ inputs.run_tests == 1 }}
+      if: ${{ inputs.deploy_as != '' && inputs.run_tests == 1 }}
       shell: bash
       env:
         AWS_ACCESS_KEY_ID: ${{ inputs.s3_id }}

diff --git a/.github/workflows/BundleStaticLibs.yml b/.github/workflows/BundleStaticLibs.yml
@@ -93,7 +93,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
         run: |
-          python scripts/amalgamation.py
+          python3 scripts/amalgamation.py
           zip -j static-lib-osx-${{ matrix.architecture }}.zip src/include/duckdb.h build/release/libduckdb_bundle.a
           ./scripts/upload-assets-to-staging.sh github_release static-lib-osx-${{ matrix.architecture }}.zip
 
@@ -124,7 +124,6 @@ jobs:
 
       - uses: ./.github/actions/build_extensions
         with:
-          deploy_as: windows_amd64_mingw
           duckdb_arch: windows_amd64_mingw
           vcpkg_target_triplet: x64-mingw-static
           treat_warn_as_error: 0
@@ -153,4 +152,91 @@ jobs:
         with:
           name: duckdb-static-lib-windows-mingw
           path: |
-            static-lib-windows-mingw.zip
+            static-lib-windows-mingw.zip
+  bundle-linux-arm64-static-libs:
+    name: Linux arm64 static libs
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - name: Build
+        shell: bash
+        run: |
+          docker run                                                             \
+          -v.:/duckdb                                                            \
+          -e CC=aarch64-linux-gnu-gcc                                            \
+          -e CXX=aarch64-linux-gnu-g++                                           \
+          -e CMAKE_BUILD_PARALLEL_LEVEL=2                                        \
+          -e OVERRIDE_GIT_DESCRIBE=$OVERRIDE_GIT_DESCRIBE                        \
+          -e EXTENSION_CONFIGS='/duckdb/.github/config/bundled_extensions.cmake' \
+          -e ENABLE_EXTENSION_AUTOLOADING=1                                      \
+          -e ENABLE_EXTENSION_AUTOINSTALL=1                                      \
+          -e FORCE_WARN_UNUSED=1                                                 \
+          -e DUCKDB_PLATFORM=linux_arm64                                         \
+          ubuntu:18.04                                                           \
+          bash -c "/duckdb/scripts/setup_ubuntu1804.sh && git config --global --add safe.directory /duckdb && make bundle-library -C /duckdb"
+      - name: Deploy
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          python3 scripts/amalgamation.py
+          zip -j static-lib-linux-arm64.zip src/include/duckdb.h build/release/libduckdb_bundle.a
+          ./scripts/upload-assets-to-staging.sh github_release static-lib-linux-arm64.zip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: duckdb-static-lib-linux-arm64
+          path: |
+            static-lib-linux-arm64.zip
+  bundle-linux-amd64-static-libs:
+    name: Linux amd64 static libs
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - name: Install pytest
+        run: |
+          python3 -m pip install pytest
+      - name: Build
+        shell: bash
+        run: |
+          export PWD=`pwd`
+          docker run                                                             \
+          -v$PWD:$PWD                                                            \
+          -e CMAKE_BUILD_PARALLEL_LEVEL=2                                        \
+          -e OVERRIDE_GIT_DESCRIBE=$OVERRIDE_GIT_DESCRIBE                        \
+          -e EXTENSION_CONFIGS="$PWD/.github/config/bundled_extensions.cmake"    \
+          -e ENABLE_EXTENSION_AUTOLOADING=1                                      \
+          -e ENABLE_EXTENSION_AUTOINSTALL=1                                      \
+          -e BUILD_BENCHMARK=1                                                   \
+          -e FORCE_WARN_UNUSED=1                                                 \
+          -e DUCKDB_RUN_PARALLEL_CSV_TESTS=1                                     \
+          quay.io/pypa/manylinux2014_x86_64                                      \
+          bash -c "yum install -y perl-IPC-Cmd && git config --global --add safe.directory $PWD && make bundle-library -C $PWD"
+      - name: Print platform
+        shell: bash
+        run: ./build/release/duckdb -c "PRAGMA platform;"
+
+      - name: Deploy
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
+        run: |
+          python3 scripts/amalgamation.py
+          zip -j static-lib-linux-amd64.zip src/include/duckdb.h build/release/libduckdb_bundle.a
+          ./scripts/upload-assets-to-staging.sh github_release static-lib-linux-amd64.zip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: duckdb-static-lib-linux-amd64
+          path: |
+            static-lib-linux-amd64.zip
diff --git a/.github/workflows/LinuxRelease.yml b/.github/workflows/LinuxRelease.yml
@@ -123,7 +123,7 @@ jobs:
       shell: bash
       if: ${{ inputs.skip_tests != 'true' }}
       run: |
-        build/release/benchmark/benchmark_runner benchmark/tpch/sf1/q01.benchmark
+        build/release/benchmark/benchmark_runner benchmark/micro/update/update_with_join.benchmark
         build/release/duckdb -c "COPY (SELECT 42) TO '/dev/stdout' (FORMAT PARQUET)" | cat
 
  linux-release-aarch64:

diff --git a/.github/workflows/R.yml b/.github/workflows/R.yml
@@ -62,7 +62,6 @@ jobs:
 
       - uses: ./.github/actions/build_extensions
         with:
-          deploy_as: windows_amd64_mingw
           duckdb_arch: windows_amd64_mingw
           vcpkg_target_triplet: x64-mingw-static
           treat_warn_as_error: 0

diff --git a/.github/workflows/SwiftRelease.yml b/.github/workflows/SwiftRelease.yml
@@ -61,5 +61,9 @@ jobs:
           cd source-repo
           export TAG_NAME=`python3 -c "import sys, os; sys.path.append(os.path.join('scripts')); import package_build; print(package_build.git_dev_version())"`
           cd ..
-          git -C updated-repo tag -a $TAG_NAME -m "Release $TAG_NAME"
-          git -C updated-repo push origin $TAG_NAME
+          if [[ $(git -C updated-repo tag -l $TAG_NAME) ]]; then
+            echo 'Tag '$TAG_NAME' already exists - skipping'
+          else
+            git -C updated-repo tag -a $TAG_NAME -m "Release $TAG_NAME"
+            git -C updated-repo push origin $TAG_NAME
+          fi
diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml
@@ -280,7 +280,6 @@ jobs:
      - uses: ./.github/actions/build_extensions
        with:
          vcpkg_target_triplet: x64-windows-static-md
-         deploy_as: windows_amd64
          treat_warn_as_error: 0
          run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
          run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}

diff --git a/extension/jemalloc/jemalloc/README.md b/extension/jemalloc/jemalloc/README.md
@@ -79,16 +79,20 @@ jemalloc_constructor(void) {
 	if (cpu_count == 0) {
 		cpu_count = duckdb_malloc_ncpus();
 	}
+	unsigned long long narenas = cpu_count / 2;
+	if (narenas == 0) {
+	    narenas = 1;
+	}
 	unsigned long long bgt_count = cpu_count / 16;
 	if (bgt_count == 0) {
 		bgt_count = 1;
 	}
 	// decay is in ms
 	unsigned long long decay = DUCKDB_JEMALLOC_DECAY * 1000;
 #ifdef DEBUG
-	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count / 2, bgt_count);
+	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, narenas, bgt_count);
 #else
-	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count / 2, bgt_count);
+	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, narenas, bgt_count);
 #endif
 	je_malloc_conf = JE_MALLOC_CONF_BUFFER;
 	malloc_init();

diff --git a/extension/jemalloc/jemalloc/src/jemalloc.c b/extension/jemalloc/jemalloc/src/jemalloc.c
@@ -4275,16 +4275,20 @@ jemalloc_constructor(void) {
 	if (cpu_count == 0) {
 		cpu_count = duckdb_malloc_ncpus();
 	}
+	unsigned long long narenas = cpu_count / 2;
+	if (narenas == 0) {
+	    narenas = 1;
+	}
 	unsigned long long bgt_count = cpu_count / 16;
 	if (bgt_count == 0) {
 		bgt_count = 1;
 	}
 	// decay is in ms
 	unsigned long long decay = DUCKDB_JEMALLOC_DECAY * 1000;
 #ifdef DEBUG
-	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count / 2, bgt_count);
+	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, narenas, bgt_count);
 #else
-	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count / 2, bgt_count);
+	snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, narenas, bgt_count);
 #endif
 	je_malloc_conf = JE_MALLOC_CONF_BUFFER;
 	malloc_init();

diff --git a/extension/parquet/include/parquet_reader.hpp b/extension/parquet/include/parquet_reader.hpp
@@ -216,12 +216,6 @@ class ParquetReader {
 	void PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t out_col_idx);
 	LogicalType DeriveLogicalType(const SchemaElement &s_ele);
 
-	template <typename... Args>
-	std::runtime_error FormatException(const string fmt_str, Args... params) {
-		return std::runtime_error("Failed to read Parquet file \"" + file_name +
-		                          "\": " + StringUtil::Format(fmt_str, params...));
-	}
-
 private:
 	unique_ptr<FileHandle> file_handle;
 };

diff --git a/extension/parquet/parquet_reader.cpp b/extension/parquet/parquet_reader.cpp
@@ -492,7 +492,8 @@ void ParquetReader::InitializeSchema(ClientContext &context) {
 	}
 	// check if we like this schema
 	if (file_meta_data->schema.size() < 2) {
-		throw FormatException("Need at least one non-root column in the file");
+		throw InvalidInputException("Failed to read Parquet file '%s': Need at least one non-root column in the file",
+		                            file_name);
 	}
 	root_reader = CreateReader(context);
 	auto &root_type = root_reader->Type();

diff --git a/src/common/types/column/column_data_collection.cpp b/src/common/types/column/column_data_collection.cpp
@@ -799,7 +799,10 @@ static bool IsComplexType(const LogicalType &type) {
 
 void ColumnDataCollection::Append(ColumnDataAppendState &state, DataChunk &input) {
 	D_ASSERT(!finished_append);
-	D_ASSERT(types == input.GetTypes());
+	{
+		auto input_types = input.GetTypes();
+		D_ASSERT(types == input_types);
+	}
 
 	auto &segment = *segments.back();
 	for (idx_t vector_idx = 0; vector_idx < types.size(); vector_idx++) {

diff --git a/src/execution/index/fixed_size_allocator.cpp b/src/execution/index/fixed_size_allocator.cpp
@@ -253,6 +253,12 @@ FixedSizeAllocatorInfo FixedSizeAllocator::GetInfo() const {
 
 	for (const auto &buffer : buffers) {
 		info.buffer_ids.push_back(buffer.first);
+
+		// Memory safety check.
+		if (buffer.first > idx_t(MAX_ROW_ID)) {
+			throw InternalException("Initializing invalid buffer ID in FixedSizeAllocator::GetInfo");
+		}
+
 		info.block_pointers.push_back(buffer.second->block_pointer);
 		info.segment_counts.push_back(buffer.second->segment_count);
 		info.allocation_sizes.push_back(buffer.second->allocation_size);
@@ -289,6 +295,12 @@ void FixedSizeAllocator::Init(const FixedSizeAllocatorInfo &info) {
 
 		// read all FixedSizeBuffer data
 		auto buffer_id = info.buffer_ids[i];
+
+		// Memory safety check.
+		if (buffer_id > idx_t(MAX_ROW_ID)) {
+			throw InternalException("Initializing invalid buffer ID in FixedSizeAllocator::Init");
+		}
+
 		auto buffer_block_pointer = info.block_pointers[i];
 		auto segment_count = info.segment_counts[i];
 		auto allocation_size = info.allocation_sizes[i];

diff --git a/src/execution/index/unbound_index.cpp b/src/execution/index/unbound_index.cpp
@@ -14,6 +14,16 @@ UnboundIndex::UnboundIndex(unique_ptr<CreateInfo> create_info, IndexStorageInfo
                            TableIOManager &table_io_manager, AttachedDatabase &db)
     : Index(create_info->Cast<CreateIndexInfo>().column_ids, table_io_manager, db), create_info(std::move(create_info)),
       storage_info(std::move(storage_info_p)) {
+
+	// Memory safety check.
+	for (idx_t info_idx = 0; info_idx < storage_info.allocator_infos.size(); info_idx++) {
+		auto &info = storage_info.allocator_infos[info_idx];
+		for (idx_t buffer_idx = 0; buffer_idx < info.buffer_ids.size(); buffer_idx++) {
+			if (info.buffer_ids[buffer_idx] > idx_t(MAX_ROW_ID)) {
+				throw InternalException("Found invalid buffer ID in UnboundIndex constructor");
+			}
+		}
+	}
 }
 
 void UnboundIndex::CommitDrop() {