页内整理
This commit is contained in:
LIm1tZz7 committed
1 parent
157372ae37
commit
03a1f2bfb2
7 files changed
+199
-3
No files matched your search
@@ -28,5 +28,5 @@
|
||||
- 改动说明只在交流中记录。
|
||||
|
||||
## 当前重点
|
||||
- 主线已完成：高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复 → M4 表结构进 WAL → M5 系统表统一命名 → M6 检查点按 LSN 部分截断 → M7 事务写写冲突检测 → M8 事务保存点 → M9 角色继承 → M10 转授权 → M11 FROM 派生表与投影别名 → M12 `WITH` / CTE → M13 比较运算符补全 → M14 ANY / ALL 量词比较 → M15 优化器直方图 → M16 子查询去相关。
|
||||
- 主线已完成：高级优化器 → 并行查询 → M1 会话隔离 → M2 快照事务 → M3 WAL 与崩溃恢复 → M4 表结构进 WAL → M5 系统表统一命名 → M6 检查点按 LSN 部分截断 → M7 事务写写冲突检测 → M8 事务保存点 → M9 角色继承 → M10 转授权 → M11 FROM 派生表与投影别名 → M12 `WITH` / CTE → M13 比较运算符补全 → M14 ANY / ALL 量词比较 → M15 优化器直方图 → M16 子查询去相关 → M17 页内完整 compact。
|
||||
- 进度与计数见 `agent_credientials/project-plan.txt`，限制与风险见 `agent_credientials/known-issues.txt`。
|
||||
@@ -294,6 +294,11 @@ impl Catalog {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 这张表还有隐藏列字节吗
|
||||
pub fn has_dropped_columns(&self, table: &str) -> bool {
|
||||
self.tables.get(table).is_some_and(|e| !e.dropped_columns.is_empty())
|
||||
}
|
||||
|
||||
/// 还有隐藏列字节待整理的表
|
||||
pub fn dropped_tables(&self) -> Vec<String> {
|
||||
self.tables
|
||||
|
||||
@@ -199,6 +199,55 @@ impl HeapPage {
|
||||
pub fn iter_slots(page: &Page) -> Vec<u16> {
|
||||
(0..Self::slot_count(page)).collect()
|
||||
}
|
||||
|
||||
/// 页内死空间字节数：行区里除活行以外的部分
|
||||
pub fn dead_bytes(page: &Page) -> usize {
|
||||
let n = Self::slot_count(page);
|
||||
let mut live = 0;
|
||||
for i in 0..n as usize {
|
||||
let slot_off = SLOT_DIR_START + i * SLOT_SIZE;
|
||||
live += read_u32(&page.data, slot_off + 4) as usize;
|
||||
}
|
||||
(page.data.len() - Self::free_end(page)).saturating_sub(live)
|
||||
}
|
||||
|
||||
/// 页内整理：活行挪到页尾，槽位号不动，返回回收字节数
|
||||
pub fn compact_page(page: &mut Page) -> usize {
|
||||
let reclaimed = Self::dead_bytes(page);
|
||||
if reclaimed == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let n = Self::slot_count(page);
|
||||
let mut live: Vec<(usize, Vec<u8>)> = Vec::new();
|
||||
for i in 0..n as usize {
|
||||
let slot_off = SLOT_DIR_START + i * SLOT_SIZE;
|
||||
let off = read_u32(&page.data, slot_off) as usize;
|
||||
let len = read_u32(&page.data, slot_off + 4) as usize;
|
||||
if len > 0 {
|
||||
live.push((i, page.data[off..off + len].to_vec()));
|
||||
} else {
|
||||
write_u32(&mut page.data, slot_off, 0);
|
||||
}
|
||||
}
|
||||
|
||||
let mut end = page.data.len();
|
||||
for (i, bytes) in live {
|
||||
end -= bytes.len();
|
||||
page.data[end..end + bytes.len()].copy_from_slice(&bytes);
|
||||
write_u32(&mut page.data, SLOT_DIR_START + i * SLOT_SIZE, end as u32);
|
||||
}
|
||||
reclaimed
|
||||
}
|
||||
|
||||
/// 死空间超过四分之一就整理这一页
|
||||
pub fn compact_if_fragmented(page: &mut Page) -> usize {
|
||||
if Self::dead_bytes(page) * 4 >= page.data.len() {
|
||||
Self::compact_page(page)
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct NamedIndex {
|
||||
@@ -442,6 +491,7 @@ impl HeapTable {
|
||||
let last_id = n - 1;
|
||||
let mut slot = None;
|
||||
self.file.update_page(last_id, |p| {
|
||||
HeapPage::compact_page(p);
|
||||
slot = HeapPage::insert_tuple(p, &tuple);
|
||||
})?;
|
||||
if let Some(s) = slot {
|
||||
@@ -556,7 +606,13 @@ impl HeapTable {
|
||||
/// 按位置删一行并返回它
|
||||
pub fn delete_at(&mut self, page_id: PageId, slot: u16) -> io::Result<Option<Row>> {
|
||||
let row = self.read_at(page_id, slot)?;
|
||||
let existed = self.file.update_page(page_id, |p| HeapPage::delete_tuple(p, slot))?;
|
||||
let existed = self.file.update_page(page_id, |p| {
|
||||
let ok = HeapPage::delete_tuple(p, slot);
|
||||
if ok {
|
||||
HeapPage::compact_if_fragmented(p);
|
||||
}
|
||||
ok
|
||||
})?;
|
||||
if !existed {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -702,6 +758,19 @@ impl HeapTable {
|
||||
self.file.flush()
|
||||
}
|
||||
|
||||
/// 整理整张表的页内碎片，返回回收字节数
|
||||
pub fn compact_pages(&mut self) -> io::Result<usize> {
|
||||
let n = self.file.num_pages()?;
|
||||
let mut reclaimed = 0;
|
||||
for i in 0..n {
|
||||
if self.file.with_page(i, HeapPage::dead_bytes)? == 0 {
|
||||
continue;
|
||||
}
|
||||
reclaimed += self.file.update_page(i, HeapPage::compact_page)?;
|
||||
}
|
||||
Ok(reclaimed)
|
||||
}
|
||||
|
||||
/// 当前缓存页数
|
||||
pub fn cached_pages(&self) -> usize { self.file.cached_pages() }
|
||||
/// 命中次数
|
||||
|
||||
@@ -817,8 +817,22 @@ impl Server {
|
||||
self.write_via_wal(&op)
|
||||
}
|
||||
|
||||
/// 整理一张表：按当前列定义重写堆页，回收隐藏列占的空间
|
||||
/// 整理一张表：原地回收页内碎片，有隐藏列时重写堆页
|
||||
fn compact_table(&self, table: &str) -> std::io::Result<()> {
|
||||
let rewrite = {
|
||||
let c = self.catalog.lock().map_err(|_| std::io::Error::other("catalog lock poisoned"))?;
|
||||
c.has_dropped_columns(table)
|
||||
};
|
||||
if !rewrite {
|
||||
let reclaimed = self.with_existing_table(table, |t| {
|
||||
let n = t.compact_pages()?;
|
||||
t.flush()?;
|
||||
Ok(n)
|
||||
})?;
|
||||
log_info!(core, "compact {} reclaimed {} bytes", table, reclaimed);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let rows = self.with_existing_table(table, |t| t.scan())?;
|
||||
let op = WalOp::Compact {
|
||||
table: table.to_string(),
|
||||
|
||||
@@ -304,3 +304,77 @@ fn secondary_index_is_maintained() {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/// 整表整理要收回删行留下的页内空洞
|
||||
#[test]
|
||||
fn compact_pages_reclaims_dead_space() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("t.db");
|
||||
let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
|
||||
|
||||
let pad = "x".repeat(200);
|
||||
for i in 0..10 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
}
|
||||
let pages = t.num_pages().unwrap();
|
||||
|
||||
t.delete_by_keys(&[3]).unwrap();
|
||||
assert_eq!(t.num_pages().unwrap(), pages, "删一行不该动页数");
|
||||
let reclaimed = t.compact_pages().unwrap();
|
||||
assert!(reclaimed > 0, "整理要回收删掉的 {} 字节", 200);
|
||||
assert_eq!(t.compact_pages().unwrap(), 0, "整理过就没有空洞了");
|
||||
|
||||
let rows = t.scan().unwrap();
|
||||
assert_eq!(rows.len(), 9);
|
||||
assert!(!rows.iter().any(|r| r["id"] == json!(3)));
|
||||
assert_eq!(rows[0]["pad"], json!(pad));
|
||||
}
|
||||
|
||||
/// 删行的页内空间要能被后续插入复用
|
||||
#[test]
|
||||
fn deletes_compact_pages_so_space_is_reused() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("t.db");
|
||||
let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap();
|
||||
|
||||
let pad = "y".repeat(80);
|
||||
for i in 0..30 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
}
|
||||
assert_eq!(t.num_pages().unwrap(), 1, "30 行要先塞进一页");
|
||||
|
||||
// 删掉中间的行，只留最靠页尾的那一行
|
||||
let keys: Vec<i64> = (10..29).collect();
|
||||
t.delete_by_keys(&keys).unwrap();
|
||||
|
||||
for i in 100..119 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
}
|
||||
assert_eq!(t.num_pages().unwrap(), 1, "删掉的空间要能被复用");
|
||||
assert_eq!(t.scan().unwrap().len(), 30);
|
||||
}
|
||||
|
||||
/// 页内整理之后索引里的位置仍然指向原来的行
|
||||
#[test]
|
||||
fn compact_pages_keeps_index_positions_valid() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data = dir.path().join("t.db");
|
||||
let index = dir.path().join("t.idx");
|
||||
let mut t = HeapTable::open_indexed(&data, &index, DEFAULT_PAGE_SIZE, 4, DEFAULT_POOL_SIZE).unwrap();
|
||||
|
||||
let pad = "z".repeat(80);
|
||||
for i in 0..30 {
|
||||
t.insert_row(&row(&[("id", json!(i)), ("pad", json!(pad))])).unwrap();
|
||||
}
|
||||
let keys: Vec<i64> = (5..25).collect();
|
||||
t.delete_by_keys(&keys).unwrap();
|
||||
t.compact_pages().unwrap();
|
||||
|
||||
for i in [0, 1, 2, 3, 4, 25, 26, 27, 28, 29] {
|
||||
let found = t.get_by_column_key("id", i).unwrap();
|
||||
assert_eq!(found.as_ref().map(|r| r["pad"].clone()), Some(json!(pad)), "整理后 id={} 仍要能点查", i);
|
||||
}
|
||||
for i in 5..25 {
|
||||
assert!(t.get_by_column_key("id", i).unwrap().is_none(), "删掉的行不该再查出来");
|
||||
}
|
||||
}
|
||||
@@ -480,6 +480,38 @@ fn startup_compaction_reclaims_dropped_columns() -> Result<(), Box<dyn std::erro
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 没有隐藏列时整理表只原地回收页内空间
|
||||
#[test]
|
||||
fn compact_defragments_pages_without_dropped_columns() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (_srv, data) = start_server();
|
||||
let mut c = connect()?;
|
||||
let pad = "p".repeat(200);
|
||||
for id in 1..=5 {
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":id,"note":pad}}))?;
|
||||
}
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":6,"note":"doomed-row-marker"}}))?;
|
||||
for id in 7..=11 {
|
||||
request_ok(&mut c, serde_json::json!({"method":"insert","table":"items","row":{"id":id,"note":pad}}))?;
|
||||
}
|
||||
request_ok(&mut c, serde_json::json!({"method":"delete_keys","table":"items","keys":[6]}))?;
|
||||
|
||||
let heap = data.path().join("databases/main/items.db");
|
||||
assert!(
|
||||
std::fs::read(&heap)?.windows(17).any(|w| w == &b"doomed-row-marker"[..]),
|
||||
"删行只留墓碑，字节还在页里"
|
||||
);
|
||||
request_ok(&mut c, serde_json::json!({"method":"compact","table":"items"}))?;
|
||||
assert!(
|
||||
!std::fs::read(&heap)?.windows(17).any(|w| w == &b"doomed-row-marker"[..]),
|
||||
"原地整理要把墓碑字节收掉"
|
||||
);
|
||||
assert_eq!(std::fs::metadata(data.path().join("databases/main/wal.log"))?.len(), 0);
|
||||
|
||||
let rows: serde_json::Value = serde_json::from_str(&send(&mut c, r#"{"method":"scan","table":"items"}"#))?;
|
||||
assert_eq!(rows["rows"].as_array().map(Vec::len), Some(10), "rows survive compaction: {}", rows);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 非法库名被拒；system 是唯一保留库
|
||||
#[test]
|
||||
fn databases_reject_unsafe_names_and_keep_only_system_reserved() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
@@ -61,6 +61,8 @@ chusql-server
|
||||
|
||||
WAL 负责崩溃恢复：每条记录带自增 LSN，一组写操作与一条提交标记一次落盘；每组提交在数据和字典都刷盘之后把 `wal.checkpoint` 推进到这条提交的 LSN，含义是「到此为止的改动都已落盘」，再按这个边界丢掉日志前缀——所以日志里通常只有当组在飞的记录。进程重启时只重放检查点之后的记录，按标记分组取已提交的组、丢弃没有标记的整组，重放前逐条做幂等检查作为第二道保险，结束后再把检查点推到最后一条提交。LSN 从检查点与文件尾里较大的那个续号，跨重启单调，`SystemStatus` 应答里的 `last_lsn` 报告该检查点 LSN。建表、删表、建索引、删索引与数据改动走同一条 WAL，恢复时一并重放。
|
||||
|
||||
堆页的空间回收分两级：删行只在槽位目录里把该槽的长度置零（墓碑），页内留下的空洞按四分之一阈值判定——死空间达到整页四分之一时随这次删除一起整理（同一次写页，不额外写 WAL），把活行重新码放到页尾并原地更新槽里的偏移。`compact` 请求在没有隐藏列的表上就是逐页做同一件事，只有带隐藏列（`DROP COLUMN` 留下的旧字节）时才整表重写。整理只改页内偏移、不动槽位号，所以索引里存的「页号加槽号」位置继续有效，不需要重建索引；跨页的空洞不做填充，行仍按插入顺序进页，没有 `ORDER BY` 时的可见顺序不变。
|
||||
|
||||
系统数据库及部分管理操作仅允许管理员访问。系统表统一用 `__system_` 前缀：账号表 `__system_users` 对语句通道隐藏，角色、授权、带 option 的授权、成员四张表（`__system_roles` / `__system_grants` / `__system_grant_options` / `__system_members`）只在 `system` 库里可见，权限服务自身也通过 SQL 读写它们；存储启动时把旧名 `sys_roles` / `sys_grants` / `sys_members` 连同数据与索引文件一起迁移过来。角色支持互相继承：`GRANT 角色 A TO 角色 B` 记一条 B 属于 A 的成员关系，判定权限时沿成员关系算出向上闭包再取并集；写入时拒绝成环与自继承，删角色会清掉它两个方向的成员关系。带 `WITH GRANT OPTION` 的授权记在第四张表 `__system_grant_options` 里（同样的角色、权限、对象三列，另开一张表省掉给 `__system_grants` 加列带来的系统表结构迁移）：普通身份只要手里那条授权带 grant option 就能把同一项权限转授给别的角色，`REVOKE` 仍然只允许管理员，收回授权时连它的 grant option 一起收掉，但不级联收回别人已经转授出去的授权。授权命令由权限服务在会话层之外直接执行，`GRANT` 命令会先按当前库把对象补全成 `库.表` 再落盘，因此判断转授权资格与实际写入用的是同一个对象名。
|
||||
|
||||
服务监听地址、端口和请求限制可在 `chusql.toml` 的 `[server]` 分区中配置。
|
||||
Reference in new issue
Block a user