From 45d833afe56ee17ba6cbb2e2b7a0661fcb2be880 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:24:32 +0300 Subject: [PATCH 1/3] remove wrong deps --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac60baf..82e692c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,8 @@ smart-default = "0.7.1" log = "0.4.29" # S3 support (optional) -rust-s3 = { version = "0.37", optional = true } -aws-creds = { version = "0.39", optional = true } +rust-s3 = { version = "0.37", optional = true, default-features = false, features = ["tokio-rustls-tls"] } +aws-creds = { version = "0.39", optional = true, default-features = false, features = ["rustls-tls"] } aws-region = { version = "0.28", optional = true } walkdir = { version = "2", optional = true } From b8e4b7e936e5ac32f0b9fb7803af463c7bcc1707 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 6 Apr 2026 14:36:58 +0300 Subject: [PATCH 2/3] update --- .gitignore | 8 +++++--- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 9d11d1e..aa8da64 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ target Cargo.lock *.DS_Store -tests/data/sync/ -/tests/data/unsized_secondary_sync/ -/tests/data/unsized_primary_sync/ +tests/data/* +!tests/data/expected/ +!tests/data/persist_index_table_of_contents.wt.idx + +.claude diff --git a/Cargo.toml b/Cargo.toml index 82e692c..e28177b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "examples", "performance_measurement", "performance_measur [package] name = "worktable" -version = "0.9.0-alpha5" +version = "0.9.0-alpha6" edition = "2024" authors = ["Handy-caT"] license = "MIT" From 40954638f15b9b90edcbd5bd7ad96f7dcf83a8e9 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 6 Apr 2026 17:22:44 +0300 Subject: [PATCH 3/3] bump --- Cargo.toml | 43 ++++++++++++++++++-------------------- src/features/s3_support.rs | 14 +++++++++---- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e28177b..ebf7e2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "examples", "performance_measurement", "performance_measur [package] name = "worktable" -version = "0.9.0-alpha6" +version = "0.9.0-alpha7" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -17,40 +17,37 @@ s3-support = ["dep:rust-s3", "dep:aws-creds", "dep:aws-region", "dep:walkdir", " # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -worktable_codegen = { path = "codegen", version = "=0.9.0-alpha4" } - async-trait = "0.1.89" -eyre = "0.6.12" -derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } -tokio = { version = "1", features = ["full"] } -tracing = "0.1" -rkyv = { version = "0.8.9", features = ["uuid-1"] } -lockfree = { version = "0.5.1" } -fastrand = "2.3.0" -futures = "0.3.30" -uuid = { version = "1.10.0", features = ["v4", "v7"] } -psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } +aws-creds = { version = "0.39", optional = true, default-features = false, features = ["rustls-tls"] } +aws-region = { version = "0.28", optional = true } +convert_case = "0.6.0" data_bucket = "=0.3.12" # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } # data_bucket = { path = "../DataBucket", version = "0.3.11" } -performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } -performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } +derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } +eyre = "0.6.12" +fastrand = "2.3.0" +futures = "0.3.30" indexset = { version = "=0.14.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", path = "../indexset", version = "0.12.10", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } -convert_case = "0.6.0" +lockfree = { version = "0.5.1" } +log = "0.4.29" ordered-float = "5.0.0" parking_lot = "0.12.3" +performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } +performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } prettytable-rs = "^0.10" -smart-default = "0.7.1" -log = "0.4.29" - -# S3 support (optional) +psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } +rkyv = { version = "0.8.9", features = ["uuid-1"] } rust-s3 = { version = "0.37", optional = true, default-features = false, features = ["tokio-rustls-tls"] } -aws-creds = { version = "0.39", optional = true, default-features = false, features = ["rustls-tls"] } -aws-region = { version = "0.28", optional = true } +smart-default = "0.7.1" +tokio = { version = "1", features = ["full"] } +tracing = "0.1" +uuid = { version = "1.10.0", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } +worktable_codegen = { path = "codegen", version = "=0.9.0-alpha4" } [dev-dependencies] -rand = "0.9.1" chrono = "0.4.43" +rand = "0.9.1" diff --git a/src/features/s3_support.rs b/src/features/s3_support.rs index 4ca21f4..84aead1 100644 --- a/src/features/s3_support.rs +++ b/src/features/s3_support.rs @@ -8,13 +8,15 @@ use awsregion::Region; use s3::Bucket; use walkdir::WalkDir; -use crate::TableSecondaryIndexEventsOps; use crate::persistence::operation::{BatchOperation, Operation}; use crate::persistence::{ DiskConfig, DiskPersistenceEngine, PersistenceConfig, PersistenceEngine, SpaceDataOps, SpaceIndexOps, SpaceSecondaryIndexOps, }; -use crate::prelude::{PrimaryKeyGeneratorState, TablePrimaryKey}; +use crate::prelude::{ + PrimaryKeyGeneratorState, TablePrimaryKey, WT_DATA_EXTENSION, WT_INDEX_EXTENSION, +}; +use crate::TableSecondaryIndexEventsOps; #[derive(Debug, Clone)] pub struct S3Config { @@ -142,8 +144,6 @@ where .ok_or_else(|| eyre::eyre!("Invalid table path"))?; let s3_key = Self::full_s3_path(prefix, &relative.to_string_lossy(), table_name); - println!("Syncing S3 to {}", s3_key); - println!("File {} {}", local_path.display(), s3_key); tracing::debug!(local_path = %local_path.display(), s3_key = %s3_key, "Uploading file to S3"); let content = tokio::fs::read(local_path).await?; @@ -190,6 +190,12 @@ where let filename = s3_key.rsplit('/').next().unwrap_or(s3_key); + if !filename.ends_with(WT_DATA_EXTENSION) && !filename.ends_with(WT_INDEX_EXTENSION) + { + tracing::debug!(s3_key = %s3_key, "Skipping non-table file"); + continue; + } + let local_path = table_path.join(filename); tracing::debug!(s3_key = %s3_key, local_path = %local_path.display(), "Downloading file from S3");