8000 bump vss, handle reverting append when index is unknown by Maxxen · Pull Request #11681 · duckdb/duckdb · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

bump vss, handle reverting append when index is unknown #11681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 17, 2024
Merged

bump vss, handle reverting append when index is unknown #11681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f686803
bump vss
Maxxen Apr 16, 2024
1f063ea
Merge branch 'main' into bump-vss
Maxxen Apr 16, 2024
0528a0f
catch errors when vacuuming index, dont try to rever unknown indexes
Maxxen Apr 16, 2024
49099e8
handle updates, deletes, more index initialization
Maxxen Apr 16, 2024
b33810e
format
Maxxen Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/config/extensions.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ aws,https://github.com/duckdb/duckdb_aws,f7b8729f1cce5ada5d4add70e1486de50763fb9
azure,https://github.com/duckdb/duckdb_azure,09623777a366572bfb8fa53e47acdf72133a360e,
spatial,https://github.com/duckdb/duckdb_spatial,8ac803e986ccda34f32dee82a7faae95b72b3492,
iceberg,https://github.com/duckdb/duckdb_iceberg,d89423c2ff90a0b98a093a133c8dfe2a55b9e092,
vss,https://github.com/duckdb/duckdb_vss,9038b50cefb8bfd6b8ab8a254d3c728f3f172d15,
vss,https://github.com/duckdb/duckdb_vss,8145f41d97178e82bed3376215eb8d02bcf1eec5,
2 changes: 1 addition & 1 deletion .github/config/out_of_tree_extensions.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,6 @@ endif()
duckdb_extension_load(vss
LOAD_TESTS
GIT_URL https://github.com/duckdb/duckdb_vss
GIT_TAG 9038b50cefb8bfd6b8ab8a254d3c728f3f172d15
GIT_TAG 8145f41d97178e82bed3376215eb8d02bcf1eec5
TEST_DIR test/sql
)
26 changes: 13 additions & 13 deletions src/execution/index/unknown_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,44 +22,44 @@ string UnknownIndex::GenerateErrorMessage() const {
}

ErrorData UnknownIndex::Append(IndexLock &, DataChunk &, Vector &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::VerifyAppend(DataChunk &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::VerifyAppend(DataChunk &, ConflictManager &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::CommitDrop(IndexLock &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::Delete(IndexLock &, DataChunk &, Vector &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
ErrorData UnknownIndex::Insert(IndexLock &, DataChunk &, Vector &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
IndexStorageInfo UnknownIndex::GetStorageInfo(bool) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
bool UnknownIndex::MergeIndexes(IndexLock &, Index &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::Vacuum(IndexLock &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
idx_t UnknownIndex::GetInMemorySize(IndexLock &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
void UnknownIndex::CheckConstraintsForChunk(DataChunk &, ConflictManager &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}
string UnknownIndex::VerifyAndToString(IndexLock &, bool) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}

string UnknownIndex::GetConstraintViolationMessage(VerifyExistenceType, idx_t, DataChunk &) {
throw NotImplementedException(GenerateErrorMessage());
throw MissingExtensionException(GenerateErrorMessage());
}

} // namespace duckdb
5 changes: 3 additions & 2 deletions src/include/duckdb/storage/table/data_table_info.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ class TableIOManager;
struct DataTableInfo {
DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table);

//! Initialize any unknown indexes whose types might now be present after an extension load
void InitializeIndexes(ClientContext &context);
//! Initialize any unknown indexes whose types might now be present after an extension load, optionally throwing an
//! exception if an index can't be initialized
void InitializeIndexes(ClientContext &context, bool throw_on_failure = false);

//! The database instance of the table
AttachedDatabase &db;
Expand Down
5 changes: 3 additions & 2 deletions src/include/duckdb/storage/table/table_index_list.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ class TableIndexList {
void CommitDrop(const string &name);
//! Returns true, if the index name does not exist
bool NameIsUnique(const string &name);
//! Initializes unknown indexes that might now be present after an extension load
void InitializeIndexes(ClientContext &context, DataTableInfo &table_info);
//! Initializes unknown indexes that might now be present after an extension load, optionally throwing an exception
//! if a index cant be initialized
void InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure = false);
bool Empty();
idx_t Count();
void Move(TableIndexList &other);
Expand Down
17 changes: 13 additions & 4 deletions src/storage/data_table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ DataTableInfo::DataTableInfo(AttachedDatabase &db, shared_ptr<TableIOManager> ta
table(std::move(table)) {
}

void DataTableInfo::InitializeIndexes(ClientContext &context) {
indexes.InitializeIndexes(context, *this);
void DataTableInfo::InitializeIndexes(ClientContext &context, bool throw_on_failure) {
indexes.InitializeIndexes(context, *this, throw_on_failure);
}

bool DataTableInfo::IsTemporary() const {
Expand Down Expand Up @@ -860,7 +860,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
row_data[i] = current_row_base + i;
}
info->indexes.Scan([&](Index &index) {
index.Delete(chunk, row_identifiers);
if (!index.IsUnknown()) {
index.Delete(chunk, row_identifiers);
}
return false;
});
current_row_base += chunk.size();
Expand All @@ -870,7 +872,9 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
// we need to vacuum the indexes to remove any buffers that are now empty
// due to reverting the appends
info->indexes.Scan([&](Index &index) {
index.Vacuum();
if (!index.IsUnknown()) {
index.Vacuum();
}
return false;
});

Expand Down Expand Up @@ -1002,6 +1006,8 @@ idx_t DataTable::Delete(TableCatalogEntry &table, ClientContext &context, Vector
return 0;
}

info->InitializeIndexes(context, true);

auto &transaction = DuckTransaction::Get(context, db);
auto &local_storage = LocalStorage::Get(transaction);
bool has_delete_constraints = TableHasDeleteConstraints(table);
Expand Down Expand Up @@ -1159,6 +1165,9 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
throw TransactionException("Transaction conflict: cannot update a table that has been altered!");
}

// check that there are no unknown indexes
info->InitializeIndexes(context, true);

// first verify that no constraints are violated
VerifyUpdateConstraints(context, table, updates, column_ids);

Expand Down
6 changes: 5 additions & 1 deletion src/storage/local_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,11 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
// we need to vacuum the indexes to remove any buffers that are now empty
// due to reverting the appends
table.info->indexes.Scan([&](Index &index) {
index.Vacuum();
try {
index.Vacuum();
} catch (std::exception &ex) { // LCOV_EXCL_START
error = ErrorData(ex);
} // LCOV_EXCL_STOP
return false;
});
error.Throw();
Expand Down
7 changes: 6 additions & 1 deletion src/storage/table_index_list.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ bool TableIndexList::NameIsUnique(const string &n D3D2 ame) {
return true;
}

void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info) {
void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &table_info, bool throw_on_failure) {
lock_guard<mutex> lock(indexes_lock);
for (auto &index : indexes) {
if (!index->IsUnknown()) {
Expand All @@ -68,6 +68,11 @@ void TableIndexList::InitializeIndexes(ClientContext &context, DataTableInfo &ta
// Do we know the type of this index now?
auto index_type = context.db->config.GetIndexTypes().FindByName(index_type_name);
if (!index_type) {
if (throw_on_failure) {
throw MissingExtensionException(
"Cannot initialize index '%s', unknown index type '%s'. You probably need to load an extension.",
unknown_index.name, index_type_name);
}
continue;
}

Expand Down
4 changes: 3 additions & 1 deletion src/transaction/undo_buffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,9 @@ void UndoBuffer::Cleanup() {
// possibly vacuum indexes
for (const auto &table : state.indexed_tables) {
table.second->info->indexes.Scan([&](Index &index) {
index.Vacuum();
if (!index.IsUnknown()) {
index.Vacuum();
}
return false;
});
}
Expand Down
0