From 03a1f2bfb20bea5db4cdff89dba0b7dbb0d98b14 Mon Sep 17 00:00:00 2001 From: LIm1tZz7 <36764786+TheBadRoger@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:18:52 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A1=B5=E5=86=85=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- chusql-core/storage/src/catalog.rs | 5 ++ chusql-core/storage/src/heap.rs | 71 +++++++++++++++++++++- chusql-core/storage/src/runtime.rs | 16 ++++- chusql-core/storage/tests/heap_test.rs | 74 +++++++++++++++++++++++ chusql-core/storage/tests/runtime_test.rs | 32 ++++++++++ docs/architecture.md | 2 + 7 files changed, 199 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1bf57d4..7652a16 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,5 +28,5 @@ - 改动说明只在交流中记录。 ## 当前重点 -- 主线已完成：高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复 → M4 表结构进 WAL → M5 系统表统一命名 → M6 检查点按 LSN 部分截断 → M7 事务写写冲突检测 → M8 事务保存点 → M9 角色继承 → M10 转授权 → M11 FROM 派生表与投影别名 → M12 `WITH` / CTE → M13 比较运算符补全 → M14 ANY / ALL 量词比较 → M15 优化器直方图 → M16 子查询去相关。 +- 主线已完成：高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复 → M4 表结构进 WAL → M5 系统表统一命名 → M6 检查点按 LSN 部分截断 → M7 事务写写冲突检测 → M8 事务保存点 → M9 角色继承 → M10 转授权 → M11 FROM 派生表与投影别名 → M12 `WITH` / CTE → M13 比较运算符补全 → M14 ANY / ALL 量词比较 → M15 优化器直方图 → M16 子查询去相关 → M17 页内完整 compact。 - 进度与计数见 `agent_credientials/project-plan.txt`，限制与风险见 `agent_credientials/known-issues.txt`。 diff --git a/chusql-core/storage/src/catalog.rs b/chusql-core/storage/src/catalog.rs index 9869a96..2e69324 100644 --- a/chusql-core/storage/src/catalog.rs +++ b/chusql-core/storage/src/catalog.rs @@ -294,6 +294,11 @@ impl Catalog { Ok(()) } + /// 这张表还有隐藏列字节吗 + pub fn has_dropped_columns(&self, table: &str) -> bool { + self.tables.get(table).is_some_and(|e| !e.dropped_columns.is_empty()) + } + /// 还有隐藏列字节待整理的表 pub fn dropped_tables(&self) -> Vec { self.tables diff --git a/chusql-core/storage/src/heap.rs b/chusql-core/storage/src/heap.rs index 19a1e7f..7c91908 100644 --- a/chusql-core/storage/src/heap.rs +++ b/chusql-core/storage/src/heap.rs @@ -199,6 +199,55 @@ impl HeapPage { pub fn iter_slots(page: &Page) -> Vec { (0..Self::slot_count(page)).collect() } + + /// 页内死空间字节数：行区里除活行以外的部分 + pub fn dead_bytes(page: &Page) -> usize { + let n = Self::slot_count(page); + let mut live = 0; + for i in 0..n as usize { + let slot_off = SLOT_DIR_START + i * SLOT_SIZE; + live += read_u32(&page.data, slot_off + 4) as usize; + } + (page.data.len() - Self::free_end(page)).saturating_sub(live) + } + + /// 页内整理：活行挪到页尾，槽位号不动，返回回收字节数 + pub fn compact_page(page: &mut Page) -> usize { + let reclaimed = Self::dead_bytes(page); + if reclaimed == 0 { + return 0; + } + + let n = Self::slot_count(page); + let mut live: Vec<(usize, Vec)> = Vec::new(); + for i in 0..n as usize { + let slot_off = SLOT_DIR_START + i * SLOT_SIZE; + let off = read_u32(&page.data, slot_off) as usize; + let len = read_u32(&page.data, slot_off + 4) as usize; + if len > 0 { + live.push((i, page.data[off..off + len].to_vec())); + } else { + write_u32(&mut page.data, slot_off, 0); + } + } + + let mut end = page.data.len(); + for (i, bytes) in live { + end -= bytes.len(); + page.data[end..end + bytes.len()].copy_from_slice(&bytes); + write_u32(&mut page.data, SLOT_DIR_START + i * SLOT_SIZE, end as u32); + } + reclaimed + } + + /// 死空间超过四分之一就整理这一页 + pub fn compact_if_fragmented(page: &mut Page) -> usize { + if Self::dead_bytes(page) * 4 >= page.data.len() { + Self::compact_page(page) + } else { + 0 + } + } } struct NamedIndex { @@ -442,6 +491,7 @@ impl HeapTable { let last_id = n - 1; let mut slot = None; self.file.update_page(last_id, |p| { + HeapPage::compact_page(p); slot = HeapPage::insert_tuple(p, &tuple); })?; if let Some(s) = slot { @@ -556,7 +606,13 @@ impl HeapTable { /// 按位置删一行并返回它 pub fn delete_at(&mut self, page_id: PageId, slot: u16) -> io::Result> { let row = self.read_at(page_id, slot)?; - let existed = self.file.update_page(page_id, |p| HeapPage::delete_tuple(p, slot))?; + let existed = self.file.update_page(page_id, |p| { + let ok = HeapPage::delete_tuple(p, slot); + if ok { + HeapPage::compact_if_fragmented(p); + } + ok + })?; if !existed { return Ok(None); } @@ -702,6 +758,19 @@ impl HeapTable { self.file.flush() } + /// 整理整张表的页内碎片，返回回收字节数 + pub fn compact_pages(&mut self) -> io::Result { + let n = self.file.num_pages()?; + let mut reclaimed = 0; + for i in 0..n { + if self.file.with_page(i, HeapPage::dead_bytes)? == 0 { + continue; + } + reclaimed += self.file.update_page(i, HeapPage::compact_page)?; + } + Ok(reclaimed) + } + /// 当前缓存页数 pub fn cached_pages(&self) -> usize { self.file.cached_pages() } /// 命中次数 diff --git a/chusql-core/storage/src/runtime.rs b/chusql-core/storage/src/runtime.rs index 01501b7..f676b04 100644 --- a/chusql-core/storage/src/runtime.rs +++ b/chusql-core/storage/src/runtime.rs @@ -817,8 +817,22 @@ impl Server { self.write_via_wal(&op) } - /// 整理一张表：按当前列定义重写堆页，回收隐藏列占的空间 + /// 整理一张表：原地回收页内碎片，有隐藏列时重写堆页 fn compact_table(&self, table: &str) -> std::io::Result<()> { + let rewrite = { + let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?; + c.has_dropped_columns(table) + }; + if !rewrite { + let reclaimed = self.with_existing_table(table, |t| { + let n = t.compact_pages()?; + t.flush()?; + Ok(n) + })?; + log_info!(core, "compact {} reclaimed {} bytes", table, reclaimed); + return Ok(()); + } + let rows = self.with_existing_table(table, |t| t.scan())?; let op = WalOp::Compact { table: table.to_string(), diff --git a/chusql-core/storage/tests/heap_test.rs b/chusql-core/storage/tests/heap_test.rs index 7af1288..fa433c3 100644 --- a/chusql-core/storage/tests/heap_test.rs +++ b/chusql-core/storage/tests/heap_test.rs @@ -304,3 +304,77 @@ fn secondary_index_is_maintained() { ); } + +/// 整表整理要收回删行留下的页内空洞 +#[test] +fn compact_pages_reclaims_dead_space() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); + + let pad = "x".repeat(200); + for i in 0..10 { + t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap(); + } + let pages = t.num_pages().unwrap(); + + t.delete_by_keys(&[3]).unwrap(); + assert_eq!(t.num_pages().unwrap(), pages, "删一行不该动页数"); + let reclaimed = t.compact_pages().unwrap(); + assert!(reclaimed > 0, "整理要回收删掉的 {} 字节", 200); + assert_eq!(t.compact_pages().unwrap(), 0, "整理过就没有空洞了"); + + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), 9); + assert!(!rows.iter().any(|r| r["id"] == json!(3))); + assert_eq!(rows[0]["pad"], json!(pad)); +} + +/// 删行的页内空间要能被后续插入复用 +#[test] +fn deletes_compact_pages_so_space_is_reused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); + + let pad = "y".repeat(80); + for i in 0..30 { + t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap(); + } + assert_eq!(t.num_pages().unwrap(), 1, "30 行要先塞进一页"); + + // 删掉中间的行，只留最靠页尾的那一行 + let keys: Vec = (10..29).collect(); + t.delete_by_keys(&keys).unwrap(); + + for i in 100..119 { + t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap(); + } + assert_eq!(t.num_pages().unwrap(), 1, "删掉的空间要能被复用"); + assert_eq!(t.scan().unwrap().len(), 30); +} + +/// 页内整理之后索引里的位置仍然指向原来的行 +#[test] +fn compact_pages_keeps_index_positions_valid() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("t.db"); + let index = dir.path().join("t.idx"); + let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap(); + + let pad = "z".repeat(80); + for i in 0..30 { + t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap(); + } + let keys: Vec = (5..25).collect(); + t.delete_by_keys(&keys).unwrap(); + t.compact_pages().unwrap(); + + for i in [0, 1, 2, 3, 4, 25, 26, 27, 28, 29] { + let found = t.get_by_column_key("id", i).unwrap(); + assert_eq!(found.as_ref().map(|r| r["pad"].clone()), Some(json!(pad)), "整理后 id={} 仍要能点查", i); + } + for i in 5..25 { + assert!(t.get_by_column_key("id", i).unwrap().is_none(), "删掉的行不该再查出来"); + } +} diff --git a/chusql-core/storage/tests/runtime_test.rs b/chusql-core/storage/tests/runtime_test.rs index 4c64de6..b87fee6 100644 --- a/chusql-core/storage/tests/runtime_test.rs +++ b/chusql-core/storage/tests/runtime_test.rs @@ -480,6 +480,38 @@ fn startup_compaction_reclaims_dropped_columns() -> Result<(), Box Result<(), Box> { + let (_srv, data) = start_server(); + let mut c = connect()?; + let pad = "p".repeat(200); + for id in 1..=5 { + request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":id,"note":pad}}))?; + } + request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":6,"note":"doomed-row-marker"}}))?; + for id in 7..=11 { + request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":id,"note":pad}}))?; + } + request_ok(&mut c, serde_json::json!({"method":"delete_keys","table":"items","keys":[6]}))?; + + let heap = data.path().join("databases/main/items.db"); + assert!( + std::fs::read(&heap)?.windows(17).any(|w| w == &b"doomed-row-marker"[..]), + "删行只留墓碑，字节还在页里" + ); + request_ok(&mut c, serde_json::json!({"method":"compact","table":"items"}))?; + assert!( + !std::fs::read(&heap)?.windows(17).any(|w| w == &b"doomed-row-marker"[..]), + "原地整理要把墓碑字节收掉" + ); + assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0); + + let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?; + assert_eq!(rows["rows"].as_array().map(Vec::len), Some(10), "rows survive compaction: {}", rows); + Ok(()) +} + /// 非法库名被拒；system 是唯一保留库 #[test] fn databases_reject_unsafe_names_and_keep_only_system_reserved() -> Result<(), Box> { diff --git a/docs/architecture.md b/docs/architecture.md index d5f8deb..8c35aef 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,6 +61,8 @@ chusql-server WAL 负责崩溃恢复：每条记录带自增 LSN，一组写操作与一条提交标记一次落盘；每组提交在数据和字典都刷盘之后把 `wal.checkpoint` 推进到这条提交的 LSN，含义是「到此为止的改动都已落盘」，再按这个边界丢掉日志前缀——所以日志里通常只有当组在飞的记录。进程重启时只重放检查点之后的记录，按标记分组取已提交的组、丢弃没有标记的整组，重放前逐条做幂等检查作为第二道保险，结束后再把检查点推到最后一条提交。LSN 从检查点与文件尾里较大的那个续号，跨重启单调，`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。建表、删表、建索引、删索引与数据改动走同一条 WAL，恢复时一并重放。 +堆页的空间回收分两级：删行只在槽位目录里把该槽的长度置零（墓碑），页内留下的空洞按四分之一阈值判定——死空间达到整页四分之一时随这次删除一起整理（同一次写页，不额外写 WAL），把活行重新码放到页尾并原地更新槽里的偏移。`compact` 请求在没有隐藏列的表上就是逐页做同一件事，只有带隐藏列（`DROP COLUMN` 留下的旧字节）时才整表重写。整理只改页内偏移、不动槽位号，所以索引里存的「页号加槽号」位置继续有效，不需要重建索引；跨页的空洞不做填充，行仍按插入顺序进页，没有 `ORDER BY` 时的可见顺序不变。 + 系统数据库及部分管理操作仅允许管理员访问。系统表统一用 `__system_` 前缀：账号表 `__system_users` 对语句通道隐藏，角色、授权、带 option 的授权、成员四张表（`__system_roles` / `__system_grants` / `__system_grant_options` / `__system_members`）只在 `system` 库里可见，权限服务自身也通过 SQL 读写它们；存储启动时把旧名 `sys_roles` / `sys_grants` / `sys_members` 连同数据与索引文件一起迁移过来。角色支持互相继承：`GRANT 角色 A TO 角色 B` 记一条 B 属于 A 的成员关系，判定权限时沿成员关系算出向上闭包再取并集；写入时拒绝成环与自继承，删角色会清掉它两个方向的成员关系。带 `WITH GRANT OPTION` 的授权记在第四张表 `__system_grant_options` 里（同样的角色、权限、对象三列，另开一张表省掉给 `__system_grants` 加列带来的系统表结构迁移）：普通身份只要手里那条授权带 grant option 就能把同一项权限转授给别的角色，`REVOKE` 仍然只允许管理员，收回授权时连它的 grant option 一起收掉，但不级联收回别人已经转授出去的授权。授权命令由权限服务在会话层之外直接执行，`GRANT` 命令会先按当前库把对象补全成 `库.表` 再落盘，因此判断转授权资格与实际写入用的是同一个对象名。 服务监听地址、端口和请求限制可在 `chusql.toml` 的 `[server]` 分区中配置。