8000 Improve single table catalogs by samansmink · Pull Request #155 · duckdb/duckdb-delta · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Improve single table catalogs #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 11, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/delta_extension.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ static unique_ptr<Catalog> DeltaCatalogAttach(StorageExtensionInfo *storage_info
}
}

res->SetDefaultTable(DEFAULT_SCHEMA, DEFAULT_DELTA_TABLE);
res->SetDefaultTable(DEFAULT_SCHEMA, name);

return std::move(res);
}
Expand Down
2 changes: 0 additions & 2 deletions src/include/delta_extension.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

#include "duckdb.hpp"

#define DEFAULT_DELTA_TABLE "delta_table"

namespace duckdb {

class DeltaExtension : public Extension {
Expand Down
3 changes: 3 additions & 0 deletions src/include/storage/delta_schema_entry.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

namespace duckdb {
class DeltaTransaction;
class DeltaCatalog;

class DeltaSchemaEntry : public SchemaCatalogEntry {
public:
Expand Down Expand Up @@ -42,6 +43,8 @@ class DeltaSchemaEntry : public SchemaCatalogEntry {

optional_ptr<DeltaTableEntry> GetCachedTable();

unique_ptr<DeltaTableEntry> CreateTableEntry(ClientContext &context);

private:
//! Delta tables may be cached in the SchemaEntry. Since the TableEntry holds the snapshot, this allows sharing a
//! snapshot between different scans.
Expand Down
6 changes: 5 additions & 1 deletion src/include/storage/delta_transaction.hpp
8000
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ class DeltaTransaction : public Transaction {
};

public:
unique_ptr<DeltaTableEntry> table_entry;
optional_ptr<DeltaTableEntry> GetTableEntry();
DeltaTableEntry &InitializeTableEntry(ClientContext &context, DeltaSchemaEntry &schema_entry);

private:
mutex lock;
unique_ptr<DeltaTableEntry> table_entry;

// DeltaConnection connection;
DeltaTransactionState transaction_state;
AccessMode access_mode;
Expand Down
5 changes: 3 additions & 2 deletions src/storage/delta_catalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ optional_idx DeltaCatalog::GetCatalogVersion(ClientContext &context) {
}

// Option 2: snapshot is cached in transaction
if (delta_transaction.table_entry) {
version = delta_transaction.table_entry->snapshot->GetVersion();
auto transaction_table_entry = delta_transaction.GetTableEntry();
if (transaction_table_entry) {
version = transaction_table_entry->snapshot->GetVersion();
}

if (version != DConstants::INVALID_INDEX) {
Expand Down
24 changes: 12 additions & 12 deletions src/storage/delta_schema_entry.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ static bool CatalogTypeIsSupported(CatalogType type) {
}
}

static unique_ptr<DeltaTableEntry> CreateTableEntry(ClientContext &context, DeltaCatalog &delta_catalog,
DeltaSchemaEntry &schema_entry) {
unique_ptr<DeltaTableEntry> DeltaSchemaEntry::CreateTableEntry(ClientContext &context) {
auto &delta_catalog = catalog.Cast<DeltaCatalog>();
auto snapshot = make_shared_ptr<DeltaMultiFileList>(context, delta_catalog.GetDBPath());

// Get the names and types from the delta snapshot
Expand All @@ -114,18 +114,18 @@ static unique_ptr<DeltaTableEntry> CreateTableEntry(ClientContext &context, Delt
for (idx_t i = 0; i < return_types.size(); i++) {
table_info.columns.AddColumn(ColumnDefinition(names[i], return_types[i]));
}
table_info.table = DEFAULT_DELTA_TABLE;
auto table_entry = make_uniq<DeltaTableEntry>(delta_catalog, schema_entry, table_info);
table_info.table = delta_catalog.GetName();
auto table_entry = make_uniq<DeltaTableEntry>(delta_catalog, *this, table_info);
table_entry->snapshot = std::move(snapshot);

return table_entry;
}

void DeltaSchemaEntry::Scan(ClientContext &context, CatalogType type,
const std::function<void(CatalogEntry &)> &callback) {
if (!CatalogTypeIsSupported(type)) {
if (CatalogTypeIsSupported(type)) {
auto transaction = catalog.GetCatalogTransaction(context);
auto default_table = GetEntry(transaction, type, DEFAULT_DELTA_TABLE);
auto default_table = GetEntry(transaction, type, catalog.GetName());
if (default_table) {
callback(*default_table);
}
Expand All @@ -146,24 +146,24 @@ optional_ptr<CatalogEntry> DeltaSchemaEntry::GetEntry(CatalogTransaction transac
}
auto &context = transaction.GetContext();

if (type == CatalogType::TABLE_ENTRY && name == DEFAULT_DELTA_TABLE) {
if (type == CatalogType::TABLE_ENTRY && name == catalog.GetName()) {
auto &delta_transaction = GetDeltaTransaction(transaction);
auto &delta_catalog = catalog.Cast<DeltaCatalog>();

if (delta_transaction.table_entry) {
return *delta_transaction.table_entry;
auto transaction_table_entry = delta_transaction.GetTableEntry();
if (transaction_table_entry) {
return *transaction_table_entry;
}

if (delta_catalog.UseCachedSnapshot()) {
unique_lock<mutex> l(lock);
if (!cached_table) {
cached_table = CreateTableEntry(context, delta_catalog, *this);
cached_table = CreateTableEntry(context);
}
return *cached_table;
}

delta_transaction.table_entry = CreateTableEntry(context, delta_catalog, *this);
return *delta_transaction.table_entry;
return delta_transaction.InitializeTableEntry(context, *this);;
}

return nullptr;
Expand Down
14 changes: 14 additions & 0 deletions src/storage/delta_transaction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,18 @@ AccessMode DeltaTransaction::GetAccessMode() const {
return access_mode;
}

optional_ptr<DeltaTableEntry> DeltaTransaction::GetTableEntry() {
unique_lock<mutex> lck(lock);
return table_entry;
}

DeltaTableEntry& DeltaTransaction::InitializeTableEntry(ClientContext &context, DeltaSchemaEntry &schema_entry) {
unique_lock<mutex> lck(lock);
if (!table_entry) {
table_entry = schema_entry.CreateTableEntry(context);
}
return *table_entry;
}


} // namespace duckdb
55 changes: 20 additions & 35 deletions test/sql/dat/attach.test
C371
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,29 @@ select utf8 from dt
3
4

# We can query the table using the catalog name + the constant `delta_table` as name
query III
SELECT database_name, schema_name, table_name from duckdb_tables;
----
dt main dt

query IIIIII
show all tables;
----
dt main dt [utf8, int64, int32, int16, int8, float32, float64, bool, binary, decimal, date32, timestamp] [VARCHAR, BIGINT, INTEGER, SMALLINT, TINYINT, FLOAT, DOUBLE, BOOLEAN, BLOB, DECIMAL(5,3), DATE, TIMESTAMP WITH TIME ZONE] false

# We can query the table using the catalog name + the table name
query I
select utf8 from dt.delta_table
select utf8 from dt.dt
----
0
1
2
3
4

# We can query the table using the catalog name + default schema + the constant `delta_table` as name
# We can query the table using the catalog name + default schema + the table name
query I
select utf8 from dt.main.delta_table
select utf8 from dt.main.dt
----
0
1
Expand All @@ -49,7 +59,7 @@ create table dt as select 1 as id, 2 as utf8
statement error
from dt
----
Catalog Error: Ambiguity detected for 'dt': this could either refer to the 'Table' 'dt', or the attached catalog 'dt' which has a default table. To avoid this error, either detach the catalog and reattach under a different name, or use a fully qualified name for the 'Table': 'memory.main.dt' or for the Catalog Default Table: 'dt.main.delta_table'.
Catalog Error: Ambiguity detected for 'dt': this could either refer to the 'Table' 'dt', or the attached catalog 'dt' which has a default table. To avoid this error, either detach the catalog and reattach under a different name, or use a fully qualified name for the 'Table': 'memory.main.dt' or for the Catalog Default Table: 'dt.main.dt'.

# Join the two tables using fully qualified names
query III
Expand All @@ -60,46 +70,21 @@ SELECT
FROM
memory.main.dt as dt1
LEFT JOIN
dt.main.delta_table as dt2
dt.main.dt as dt2
ON
dt1.utf8 = dt2.utf8
----
1 2 2


# You shouldn't be doing this, but its technically possible: we mount the single-table-catalog
# You probably shouldn't be doing this, but its technically possible: we mount the single-table-catalog
statement ok
use dt

# We can still query the delta catalog default table by its name
query I
# TODO: we should fix this upstream
statement error
select utf8 from dt
----
0
1
2
3
4

# Or by the default delta table name (`delta_table`)
query I
select utf8 from delta_table
----
0
1
2
3
4

# Or by specifying the default schema
query I
select utf8 from main.delta_table
----
0
1
2
3
4
Catalog Error: Ambiguity detected for 'dt'

statement ok
USE memory
Expand Down
Loading
0