From 9e8c5f1ef078d58a8346f2f8fa796035249f8d61 Mon Sep 17 00:00:00 2001 From: LIm1tZz7 <36764786+TheBadRoger@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:21:55 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=92=8C=E6=94=B6=E5=B0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 198 ++++++++- benchmark/README.md | 93 ---- benchmark/chusql-storage/Cargo.lock | 334 ++++++++++++++ benchmark/chusql-storage/Cargo.toml | 16 + benchmark/chusql-storage/README.md | 36 -- benchmark/chusql-storage/benches/storage.rs | 353 +++++++++++++++ benchmark/fullchain/chusql-fullchain.cabal | 17 + benchmark/fullchain/src/Main.hs | 435 +++++++++++++++++++ benchmark/fullchain/stack.yaml | 5 + benchmark/fullchain/stack.yaml.lock | 12 + chusql-engine/chusql-engine.cabal | 2 + chusql-engine/src/ChuSQL/Algebra/Eval.hs | 94 +++- chusql-engine/src/ChuSQL/Algebra/Op.hs | 8 +- chusql-engine/src/ChuSQL/Algebra/Optimize.hs | 12 +- chusql-engine/src/ChuSQL/Engine.hs | 78 ++-- chusql-engine/src/ChuSQL/Model.hs | 22 +- chusql-engine/src/ChuSQL/Semantic.hs | 19 +- chusql-engine/src/ChuSQL/Storage.hs | 75 +++- chusql-engine/src/ChuSQL/Storage/IPC.hs | 260 +++++++---- chusql-engine/src/ChuSQL/Storage/Memory.hs | 19 +- chusql-engine/src/ChuSQL/Syntax/AST.hs | 8 +- chusql-engine/src/ChuSQL/Syntax/Parser.hs | 45 +- chusql-engine/test/Spec.hs | 203 +++++++-- chusql-storage/src/bin/server.rs | 399 +++++++++++++---- chusql-storage/src/btree.rs | 35 ++ chusql-storage/src/catalog.rs | 208 +++++++-- chusql-storage/src/heap.rs | 354 ++++++++++++--- chusql-storage/src/protocol.rs | 84 +++- chusql-storage/src/wal.rs | 102 ++++- chusql-storage/tests/btree_test.rs | 60 +++ chusql-storage/tests/heap_test.rs | 167 ++++++- chusql-storage/tests/pipe_test.rs | 172 +++++++- chusql-storage/tests/wal_test.rs | 5 +- 33 files changed, 3332 insertions(+), 598 deletions(-) delete mode 100644 benchmark/README.md create mode 100644 benchmark/chusql-storage/Cargo.lock create mode 100644 benchmark/chusql-storage/Cargo.toml delete mode 100644 benchmark/chusql-storage/README.md create mode 100644 benchmark/chusql-storage/benches/storage.rs create mode 100644 benchmark/fullchain/chusql-fullchain.cabal create mode 100644 benchmark/fullchain/src/Main.hs create mode 100644 benchmark/fullchain/stack.yaml create mode 100644 benchmark/fullchain/stack.yaml.lock diff --git a/README.md b/README.md index 342d116..5f46f83 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,192 @@ > 如果找到了较好的Rust+Haskell集成方法也会及时替换 # 性能基准 -**测试环境**:Windows / GHC 9.10.3 / `-O2` -**数据规模**:`users`、`orders` 各 500 行,两表配对共 25 万对 -**计时方式**:CPU 时间;每批至少 8 次迭代、累计超过 0.3 秒后固定批次,取 3 批中位数(避开时钟量化误差) -| 查询 | 结果行数 | 未优化 | 优化后 | 加速 | -| ------------------------------------------ | -------: | -------: | ----------: | --------: | -| `JOIN` + `WHERE u.age > 90` | 45 | 156.3 ms | **12.2 ms** | **12.8x** | -| `JOIN`(无 WHERE,只有投影) | 500 | 169.9 ms | 146.5 ms | 1.2x | -| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 175.8 ms | **13.7 ms** | **12.9x** | -| 单表 `WHERE age > 90`(对照组) | 45 | 0.183 ms | 0.275 ms | 0.7x | +三个基准,各管一件事,**主基准是全链路**(Haskell 引擎和 Rust 存储进程一起启动,跑真实情景): -> 跑法:`cd benchmark/chusql-engine && stack run`(可选参数:`用户行数 订单数 单批目标秒数`,默认 `500 500 0.3`) -> 最近更新:2026-09-24 +| 基准 | 位置 | 测什么 | +| ---- | ---- | ------ | +| **全链路** | `benchmark/fullchain/` | SQL 文本 → 引擎 → 命名管道 → Rust → WAL/堆表/索引 → 落盘,整条路 | +| 查询层 | `benchmark/chusql-engine/` | 只用内存数据,隔离出优化器与连接算法的效果 | +| 存储层 | `benchmark/chusql-storage/` | Rust 侧页 IO / B+ 树 / 堆表 / WAL 的微基准 | + +## 全链路:Haskell + Rust 一起跑 + +```bash +cd chusql-storage && cargo build --release # 先编译存储进程(找不到 release 会用 debug 并提醒你) +cd benchmark/fullchain +stack run chusql-fullchain # 默认 users = orders = 200 +stack run chusql-fullchain -- 1000 1000 # 换规模 +# 改过 chusql-engine 源码之后,先 stack build --force-dirty 再跑(原因见"计时怎么做"的坑四) +``` + +它做的事:造一个临时数据目录和独立管道名 → 起 Rust 存储进程 → 用引擎的 IPC 存储连上去 → +按真实情景跑一遍(建表、逐条 INSERT、全表 SELECT、过滤、JOIN、按主键点查)→ 校验行数 → 收摊。 + +**每条语句都是整条链路**:SQL 文本真的要过词法/语法分析、语义检查、关系代数,再走管道到 Rust, +经 WAL(fsync)、堆表、B+ 树、页缓冲池落到磁盘,结果再原路返回。 +计时用**墙上时钟(单调时钟)**,不是 CPU 时间——这条链路大部分工夫花在等 IO 和跨进程往返上。 + +### 实测(每条语句平均耗时) + +| 指标 | 200 × 200 | 500 × 500 | 1000 × 1000 | +| ---- | --------: | --------: | ----------: | +| 单条 `INSERT`(整条语句) | 2.66 ms | 2.86 ms | 2.88 ms | +| 同一条 `INSERT`(原始 IPC,不经引擎) | 2.50 ms | 2.85 ms | 2.77 ms | +| **多行 `INSERT`(100 行/条)** | — | — | **0.059 ms/行** | +| `SELECT * FROM users` | 0.4 ms | 1.0 ms | 1.9 ms | +| `SELECT name FROM users WHERE age > 90` | 0.3 ms | 0.8 ms | 2.7 ms | +| `SELECT ... JOIN ... WHERE` | 0.9 ms | 2.4 ms | 5.3 ms | +| `SELECT name FROM users WHERE id = k` | 0.2 ms | 0.2 ms | 0.2 ms | +| `DELETE FROM users WHERE id = k`(删 1 行) | — | — | 4.8 ms | +| `DELETE FROM users`(删全部 1000 行) | — | — | 20.2 ms | +| `SELECT ... WHERE code = k`(无索引 → 建索引后) | — | — | 1.8 ms → **0.2 ms** | + +四条结论: + +1. **写入是 fsync 说了算,和数据量无关。** 单条 INSERT 约 2.7 ms。能单独量出来的落盘成本约 1.0 ms + (写日志 0.43 + 清日志 0.42 + 表数据 flush 0.15),其余是这几次同步落盘之间的等待和跨进程往返。 + 表从 200 行长到 1000 行,单条耗时基本不动——**插一行和插一千行的落盘代价几乎一样**。 + 所以想快就得多行一起提交:`VALUES (...), (...), ...` 一条语句 100 行, + 每行 2.88 ms → **0.059 ms(约 49 倍)**。 +2. **点查是 O(1)。** 按主键查一行 0.2 ms(其中原始 IPC 索引查找 0.07 ms),1000 行时仍是 0.2 ms。 + 建在别的列上的索引也一样:`WHERE code = k` 从 1.8 ms(没有索引,退回扫描)降到 0.2 ms。 +3. **删除的代价和"删几行"无关了。** 删 1 行 4.8 ms、删全部 1000 行 20.2 ms: + 前者就贵在一次全表扫描 + 一次落盘,后者是同一笔扫描加上 1000 次 O(1) 的行级删除。 + 改之前,删 1 行也要把整张表重写一遍并重建整棵索引。 +4. **单表查询与那张表的行数成正比**(1000 行 1.9 ms,约 2 µs/行)。查询层现在的设计是把表拉进内存算, + 这个数字就是"拉 1000 行过管道 + 在内存里过一遍"的价钱。 + +## P2 这一版改了什么 + +四件事,都对应"把整表操作换成行级/批量操作": + +- **① 行级 DELETE。** B+ 树加 `delete`(惰性删除:只摘叶子里的键,不合并节点); + 堆表把槽位长度置 0(读侧本来就把"长度 0"当作没有这一行);查询层的 `DELETE` 改成 + "扫一遍算出要删哪些 id → `delete_keys` 一次删掉",不再 `replaceAll` 重写整表。 +- **② 多索引 + `CREATE INDEX` + 规划器选路。** 优化器把"单列等值条件"标成点查节点 + `Lookup 别名 表 列 值`;**到底有没有索引由存储层回答**——没有就回 `no_index`,查询层退回全表扫描。 + 这样索引元数据只存在数据字典一处,加索引不用改优化器。 + 边界写在下面:**只支持单列整数索引,而且索引列必须唯一**。 +- **③ 批量插入。** 语法支持 `INSERT INTO t (...) VALUES (...), (...), ...`:一条语句走一次 WAL、一次 fsync。 +- **④ 列统计。** 数据字典里每列记"不同值个数"(内存里按值哈希精确维护,上限 4096 之后转下界), + 随 `catalog.json` 落盘,**启动时扫一遍表重建**,所以重启后数字仍然准;插入/删除实时更新。 + +### 一个必须讲清楚的取舍:索引列必须唯一 + +原来的索引只服务 `id`,"唯一"是默认成立的前提。要做通用索引就必须回答一个问题: +**一个列上有重复值时,"走索引取一行"和"全表扫一遍再筛"还能得到同一个答案吗?** +现在不能——索引的每个键只放得下一个行位置。所以这一版的规则是: + +- 建索引时查重,有重复值就**报错、不建**(不留半成品); +- 索引列上插入重复值会被**拒绝**(老代码是"覆盖索引条目",那会让老行再也查不到)。 + +真正的解法是让 B+ 树支持重复键(键后面带上行号,再做叶子链上的范围扫描), +那要把 B+ 树的 `i64` 定长键换成内存可比较的变长字节键——和"复合键"是同一块地基, +留到单独一轮做。这条限制记在根 README 和 `projectplan.txt` 里,不是忘了。 + +> 顺带修掉一个旧 bug:点查节点原来会把**别名前缀丢掉**, +> 于是 `SELECT u.name FROM users u WHERE u.id = 1` 取不到列(投影按 `u.name` 找, +> 回来的行却是 `name`)。现在 `Lookup` 也带着别名,回来的一行和 `Scan 别名 表` 长得一样, +> 并且补了对照测试。 + +## 全链路这一版改了什么:别再为语义检查搬整个库 + +原来的 `runStatementM` 每条语句都先取一次**整库快照**(把所有表的所有行都拉过管道), +只为了给语义检查看列名。可是语义检查(`Semantic.check`)和查询优化(`relOpCols`)**只看列,一行数据都不碰**, +行数据是白搬的——而且每条语句都搬一次,于是单条语句的开销随库变大而线性增长。 + +改法:给 `MonadStorage` 加一个 `schema`(只要表结构,不要行),默认实现退化成"快照 + 抹掉行", +`IPCStorage` 覆盖成"只问数据字典要结构"(一次 `list_catalog` 往返,约 0.07 ms),`Engine.runStatementM` 改用它。 + +1000 × 1000 下的效果: + +| 指标 | 改前 | 改后 | 倍率 | +| ---- | ---: | ---: | ---: | +| `INSERT INTO users` | 3.65 ms | **2.67 ms** | 1.4x | +| `INSERT INTO orders` | 5.61 ms | **2.91 ms** | 1.9x | +| `SELECT * FROM users` | 6.7 ms | **2.0 ms** | 3.4x | +| `SELECT ... WHERE age > 90` | 6.6 ms | **1.7 ms** | 3.9x | +| `SELECT ... JOIN ... WHERE` | 11.2 ms | **4.0 ms** | 2.8x | +| `SELECT ... WHERE id = k` | 3.3 ms | **0.2 ms** | 16x | + +改完之后单条 INSERT 的耗时和"原始 IPC 插入"持平(2.67 vs 2.68 ms)—— +**引擎侧的每条语句开销已经小到量不出来**,剩下全是存储层的落盘成本。 + +## 查询层:等值连接改成哈希表 + +`Join` 原来是"两边两两配对再筛":25 万对,每对拼一次行、按列名查两次值,单对约 0.66 µs, +乘起来就是 165 ms 上下——这正是下表"未优化"那一列的全部耗时。 +现在条件是 `a = b` 这种两列等值时,先拿右表建哈希表、再拿左表逐行探测,配对次数降到 1000 次哈希查找; +认不出等值键时自动退回嵌套循环,输出顺序与原来完全一致。 + +数据规模 users / orders 各 500 行(配对 25 万对),中位数,单位 ms: + +| 查询 | 结果行数 | 改前 未优化 | 改后 未优化 | 改前 优化后 | 改后 优化后 | +| ------------------------------------------ | -------: | ----------: | ----------: | ----------: | ----------: | +| `JOIN` + `WHERE u.age > 90` | 45 | 110.4 ms | **0.267 ms** | 8.9 ms | **0.30 ms** | +| `JOIN`(无 WHERE,只有投影) | 500 | 135.7 ms | **0.397 ms** | 111.3 ms | **0.549 ms** | +| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 130.9 ms | **0.336 ms** | 9.2 ms | **0.320 ms** | +| 单表 `WHERE age > 90`(对照组) | 45 | 0.095 ms | 0.084 ms | 0.149 ms | 0.13 ms | + +**连接类查询约 300 倍;换成 2000 行 / 400 万对时是 500~900 倍**(嵌套循环是 O(左 × 右),哈希连接是线性,规模越大省得越多)。 + +需要注意:表里"优化器开/关"的比值从原来的 12~17 倍降到了 1.0 倍左右—— +**不是优化器变差了,是分母变了**:原来那十几倍几乎全来自"条件把配对次数从 25 万筛到 2.25 万", +而连接本身只要 0.4 ms 之后,下推省下的时间已不占主导。**看性能要看绝对耗时,不是比值。** + +## 存储层:写入路径 + +单条 `INSERT` 的完整流程(清 WAL → 写 WAL → 改数据 → 落盘 → 清 WAL)原来约 **3.8 ms**,现在约 **2.5 ms**: + +- **WAL 文件句柄常驻**:原来每条操作要 open/close 三次,而 append 模式下开完再 fsync 比看起来的 open 更贵; + 实测单次追加 990 µs → 507 µs。 +- **写前那次清空只截断、不单独 fsync**:紧跟其后的 `append` 会 fsync,那一次会把截断一起落下去。 + 这一项从 522~600 µs 降到 11.6 µs。 + +写入路径的瓶颈**是 fsync 次数,不是数据量**。剩下的时间基本都是"真落盘",而这是 WAL 语义要求的: +再往下压就要动"每条操作都截断 WAL"这个设计本身(真正的数据库是顺序追加 + 检查点,而不是写一条清一次)。 + +其余微基准(n = 2000):页顺序写 15~17 µs/页(含 flush)、页顺序读 4.1~4.5 µs/页、 +B+ 树插入 17.9 µs/次、点查 14.8~16.1 µs/次、堆表扫描 0.5 µs/行。 + +## 计时怎么做(四个坑都踩过了) + +**坑一:Haskell 是惰性的,不强制求值就等于没测。** +只写 `let r = evalRelOp db plan`,`r` 根本没被算出来,测到的是"定义一个变量"的时间。 +所以每次必须把结果算到底(项目里是逐行把列数累加起来当校验和,或者把行数 `evaluate` 出来)。 + +**坑二:Windows 上 CPU 时钟的精度只有约 15.6 毫秒。** +毫秒级的用例直接测会得到 `0.0 ms` 这种没意义的数字。 +查询层基准用**自适应批量**(同一用例连跑 `k` 次,`k` 翻倍到一批超过 0.3 秒再折算成单次), +把时钟误差摊薄;全链路基准因为是 IO 密集、单次就有毫秒级,直接用墙上单调时钟。 + +**坑三:机器会漂移。** 同一份代码、同一天、前后隔半小时,指标能整体差 15~25%(温度、后台进程都算)。 +所以:改前改后必须**同一时段交叉测**;小于 2 倍的差异,没交叉验证之前不要当结论。 + +**坑四(本项目的专属坑):小心链到旧引擎。** `benchmark/fullchain` 和 `chusql-engine` 在同一个 stack 项目里, +改完引擎源码后 stack 有时不会把依赖库重新拷进基准项目的安装目录——于是你测的是旧引擎, +而且数字看起来"合理"。**改过引擎就先 `stack build --force-dirty`。** +(这一条是实测踩出来的:一次"改动没效果"的排查,最后发现是引擎库的时间戳比源码还旧。) + +## 试过但没留下的(别重复踩) + +| 想法 | 结果 | 结论 | 为什么 | +| ---- | ---- | ---- | ------ | +| 给 Haskell 库加 `-O2` | 加完 8 项全部慢 15~25%;**去掉后重建再测,一样慢** | **已回退** | 这次"变慢"是机器漂移,不是 `-O2` 造成的——但回退后也没测出 `-O2` 的好处。**结论:没有证据**,要么同轮交叉测再下判断,要么别加 | +| 换掉缓冲池的 `min_by_key` 淘汰 | 池 1024 / 工作集超池:随机读 7.9 µs vs 池 16:5.1 µs | 未做 | 差距是真的(缺页时多扫 1024 个候选页),但只影响"工作集大于池"的随机读,绝对量 ~2.8 µs/次;而写入路径是 2500 µs/次,不在一个量级 | +| 把投影下推进 `Scan`(Scan 自带列清单) | 未做 | 待定 | 能消掉"无 WHERE 的 JOIN"多出来的那趟投影(约 25%,0.1 ms 级),但要改 `RelOp` 类型、`relOpCols`、Eval 和一堆测试 | +| 非唯一索引(一个键多条记录) | 未做 | 待定 | 要 B+ 树允许重复键(键后面带行号)+ 叶子链上的范围扫描;和"复合键"共用同一块地基:**键从 `i64` 定长槽位改成内存可比较的变长字节键** | +| 只给索引加"唯一"检查、不改插入语义 | 未做 | 已否 | 那样建索引时唯一、之后插重复值仍会把老行"挤"出索引,查询结果就和全表扫不一致了——宁可插入时报错 | + +## 怎么加用例 + +- **全链路**:在 `benchmark/fullchain/src/Main.hs` 的 `runScenarios` 里加 `benchSql label sql` 就行; + 要验证结果就照着 `[校验]` 那一段写断言(现在的两个校验是 `age > 90` 的行数和 JOIN 的行数)。 +- **查询层**:在 `benchmark/chusql-engine/src/Main.hs` 的 `queries` 列表里加一条 SQL 字符串,其余全自动 + (自动翻译、优化、跑两遍、比结果、算加速比)。 +- **存储层**:在 `benchmark/chusql-storage/benches/storage.rs` 里加一段 `bench(...)` / `bench_once(...)`。 + +想在文档里引用数字,记得把**数据规模、机器、时间**一起写上——脱离这些谈"快了几倍"没有意义。 + +> 最近更新:2026-09-25 diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index c8fd04e..0000000 --- a/benchmark/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# ChuSQL 基准测试 - -这个目录放**性能基准测试**,和 `chusql-engine`、`chusql-storage` 的功能代码分开, -因为它回答的问题不一样:功能代码回答"对不对",基准测试回答"快不快、快多少"。 - -``` -benchmark/ -├── README.md 本文件:怎么跑、测什么、结果怎么解读 -├── chusql-engine/ Haskell 查询层:优化器开/关的对比 -│ ├── stack.yaml 把 chusql-engine 作为本地包引入 -│ ├── chusql-bench.cabal -│ └── src/Main.hs -└── chusql-storage/ Rust 存储层:预留(见该目录下的 README) -``` - ---- - -## 怎么跑 - -```bash -cd benchmark/chusql-engine -stack run # 默认 users=500 行、orders=500 行 -stack run -- 1000 1000 0.5 # 也可以自己指定:users 行数、orders 行数、每批目标秒数 -``` - -第一次跑要编译,之后很快。整个默认跑完约 5 秒。 - ---- - -## 测什么 - -四个用例,**每个都跑两遍**:一遍走"翻译出来的原始工序单",一遍走"优化后的工序单"。 -两次的结果行数会被打印出来互相对照,**不一致会当场标出来**。 - -| 用例 | 想看什么 | -| ---- | -------- | -| `JOIN` + `WHERE u.age > 90` | 谓词下推的主场:条件能筛掉约 90% 的行 | -| `JOIN`(无 WHERE) | 只剩投影下推在起作用,收益应该温和 | -| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT` | 规则叠加时的综合效果 | -| 单表 `WHERE age > 90` | 对照组:没有下推空间,优化器只剩固定开销 | - ---- - -## 计时怎么做(两个坑都踩过了) - -**坑一:Haskell 是惰性的,不强制求值就等于没测。** -如果只写 `let r = evalRelOp db plan`,`r` 根本没被算出来,测到的是"定义一个变量"的时间。 -所以每次必须把结果**算到底**(本项目里是逐行把列数累加起来当校验和)。 - -**坑二:Windows 上 CPU 时钟的精度只有约 15.6 毫秒。** -优化后的查询只要几毫秒,直接测会得到 `0.0 ms` 这种没意义的数字。 - -所以计时用的是**自适应批量**:把同一个用例连续跑 `k` 次测一整批,`k` 不断翻倍, -直到一批累计耗时超过目标值(默认 0.3 秒),再折算成单次耗时。 -这样时钟精度的误差会被 `k` 摊薄,几毫秒的用例也能测准。 -程序会打印实际用了多大的批次(例如"优化后 64 次/批")。 - -跑正式计时之前还会**预热两遍**,避免把首次构造表数据的开销算进去。 - ---- - -## 实测结果 - -**环境**:Windows,GHC 9.10.3,`-O2` -**数据**:`users` 500 行(`id / name / age`,`age` 只有 0-99 共 100 种取值), -`orders` 500 行(`id / user_id / product`),两表配对共 25 万对。 - -| 查询 | 结果行数 | 未优化 | 优化后 | 加速 | -| ---- | -------- | ------ | ------ | ---- | -| `JOIN` + `WHERE u.age > 90` | 45 | 125.0 ms | **7.1 ms** | **17.7x** | -| `JOIN`(无 WHERE) | 500 | 132.8 ms | 105.5 ms | 1.3x | -| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 171.9 ms | **10.3 ms** | **16.8x** | -| 单表 `WHERE age > 90` | 45 | 0.111 ms | 0.153 ms | 0.7x | - -### 怎么解读 - -- **前两行的差距来自谓词下推。** 条件筛掉了约 90% 的用户,配对次数从 25 万降到约 2.25 万, - 时间直接掉一个数量级。筛得越狠,省得越多。 -- **第二行只有投影下推。** 配对次数一点没变(没有条件可下推),只是每一行变窄了, - 所以只有 1.3 倍 —— 收益温和但确实是正的。 -- **第四行反而略慢。** 单表查询本来就没有下推空间(条件早已贴着扫描), - 而优化器自己要走一遍规则,有几十微秒的固定开销(0.153 − 0.111 ≈ 0.04 ms)。 - **优化不是零成本的**:查询越大、配对越多,这笔账才越划算。 - ---- - -## 加新用例 - -在 `src/Main.hs` 的 `queries` 列表里加一条 SQL 字符串就行,其余全自动 -(会自动翻译、优化、跑两遍、比结果、算加速比)。 - -想测别的规模,直接命令行传参;想在文档里引用数字,记得把**数据规模一起写上** —— -脱离规模谈"快了几倍"是没有意义的。 diff --git a/benchmark/chusql-storage/Cargo.lock b/benchmark/chusql-storage/Cargo.lock new file mode 100644 index 0000000..ceae9ba --- /dev/null +++ b/benchmark/chusql-storage/Cargo.lock @@ -0,0 +1,334 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chusql-storage" +version = "0.1.0" +dependencies = [ + "interprocess", + "serde", + "serde_json", + "toml", +] + +[[package]] +name = "chusql-storage-bench" +version = "0.1.0" +dependencies = [ + "chusql-storage", + "serde_json", + "tempfile", +] + +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "interprocess" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/benchmark/chusql-storage/Cargo.toml b/benchmark/chusql-storage/Cargo.toml new file mode 100644 index 0000000..51fef18 --- /dev/null +++ b/benchmark/chusql-storage/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "chusql-storage-bench" +version = "0.1.0" +edition = "2024" + +# 存储层基准单独成包,不混进功能代码(见同目录 README)。 +[dependencies] +chusql-storage = { path = "../../chusql-storage" } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" + +[[bench]] +name = "storage" +harness = false diff --git a/benchmark/chusql-storage/README.md b/benchmark/chusql-storage/README.md deleted file mode 100644 index 9e539cd..0000000 --- a/benchmark/chusql-storage/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# chusql-storage 基准测试(预留) - -这个目录预留给 **Rust 存储层** 的基准测试代码,目前 `chusql-storage/` 还没有开工, -所以这里先占位。 - -## 计划放什么 - -`chusql-storage` 做的是磁盘那一层的事情,性能指标和查询层完全不同 —— -查询层关心"少算几行",存储层关心"少读几次盘"。所以基准要分开测: - -| 待测模块 | 关心的指标 | -| -------- | ---------- | -| 页管理 | 单页读写延迟、顺序读 vs 随机读吞吐 | -| B+ 树 | 插入/查找/范围扫描的耗时随数据量增长的趋势 | -| 缓冲池 | 命中率、缺页次数、替换策略(LRU 等)的对比 | -| WAL | 追加写吞吐、日志落盘延迟、恢复耗时 | -| 崩溃恢复 | 从日志重放的时间、恢复后数据一致性 | - -## 用什么 - -Rust 侧的基准测试一般用 [criterion](https://github.com/bheisler/criterion.rs): - -``` -benchmark/chusql-storage/ -├── Cargo.toml -└── benches/ - ├── page.rs - ├── btree.rs - ├── buffer_pool.rs - └── wal.rs -``` - -跑法:`cargo bench` - -另外还要和 Haskell 层对齐口径:查询层的基准是"整条查询的耗时", -存储层的基准是"单次 IO / 单次查找的耗时",两边不要混在一起比。 diff --git a/benchmark/chusql-storage/benches/storage.rs b/benchmark/chusql-storage/benches/storage.rs new file mode 100644 index 0000000..1aaf8d0 --- /dev/null +++ b/benchmark/chusql-storage/benches/storage.rs @@ -0,0 +1,353 @@ +//! 存储层基准:页 IO、B+ 树、堆表、WAL 与服务端写入路径。 +//! +//! 用 `std::time::Instant` 计时,不引第三方基准框架,离线也能跑。 +//! 每项跑 `rounds` 轮取最快一轮:存储层的抖动主要来自 fsync 与调度, +//! 取最快一轮最接近"这套代码本身要花多久"。 +//! +//! 跑法:`cargo bench`(可选环境变量 `CHUSQL_BENCH_N` 改规模,默认 2000) + +use std::time::{Duration, Instant}; + +use chusql_storage::btree::DiskBTree; +use chusql_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE}; +use chusql_storage::heap::HeapTable; +use chusql_storage::page::{Page, PageFile}; +use chusql_storage::protocol::Row; +use chusql_storage::wal::{Wal, WalOp}; +use serde_json::json; + +// 工具 +/// 跑一个操作 `ops` 次,量总耗时并折算每次耗时 +fn bench(label: &str, ops: usize, rounds: usize, mut f: F) { + let mut best = Duration::MAX; + for _ in 0..rounds { + let t0 = Instant::now(); + f(); + let d = t0.elapsed(); + if d < best { + best = d; + } + } + let per = best.as_secs_f64() * 1e6 / ops as f64; + println!( + "{:<38} {:>9.2} ms {:>10.3} us/次", + label, + best.as_secs_f64() * 1e3, + per + ); +} + +/// 只量一次(fsync 类操作太慢,不适合反复跑) +fn bench_once(label: &str, ops: usize, f: F) { + let t0 = Instant::now(); + f(); + let d = t0.elapsed(); + let per = d.as_secs_f64() * 1e6 / ops as f64; + println!( + "{:<38} {:>9.2} ms {:>10.3} us/次", + label, + d.as_secs_f64() * 1e3, + per + ); +} + +/// 造一行 +fn row(id: i64) -> Row { + let mut m = Row::new(); + m.insert("id".to_string(), json!(id)); + m.insert("name".to_string(), json!(format!("user{}", id))); + m +} + +/// 小随机数发生器,避免引 rand +fn next_rand(state: &mut u64) -> u64 { + *state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + *state >> 33 +} + +// 页 IO +/// 顺序写 / 顺序读 / 随机读 +fn bench_page(dir: &std::path::Path, n: usize) { + let path = dir.join("page.db"); + let mut pf = PageFile::with_options(&path, DEFAULT_PAGE_SIZE, 1024).unwrap(); + + bench("page 顺序写(含 flush)", n, 3, || { + pf.truncate().unwrap(); + for i in 0..n { + pf.write_page(&Page::new(i as u64, DEFAULT_PAGE_SIZE)).unwrap(); + } + pf.flush().unwrap(); + }); + drop(pf); + + // 池子故意开小,让顺序读真的去读盘 + let mut pf = PageFile::with_options(&path, DEFAULT_PAGE_SIZE, 16).unwrap(); + bench("page 顺序读(池 16,基本都缺页)", n, 3, || { + for i in 0..n { + pf.read_page(i as u64).unwrap(); + } + }); + + let mut pf = PageFile::with_options(&path, DEFAULT_PAGE_SIZE, 16).unwrap(); + let mut state = 12345u64; + bench("page 随机读(池 16)", n, 3, || { + for _ in 0..n { + let id = next_rand(&mut state) % n as u64; + pf.read_page(id).unwrap(); + } + }); + + // 池子开大、工作集比池子还大:每次缺页都要挑一个牺牲页 + let mut pf = PageFile::with_options(&path, DEFAULT_PAGE_SIZE, 1024).unwrap(); + let mut state = 999u64; + bench("page 随机读(池 1024,工作集 > 池)", n, 3, || { + for _ in 0..n { + let id = next_rand(&mut state) % n as u64; + pf.read_page(id).unwrap(); + } + }); + println!( + " (池 1024:命中 {}/{})", + pf.hits(), + pf.hits() + pf.misses() + ); +} + +// B+ 树 +/// 插入、点查、全量遍历 +fn bench_btree(dir: &std::path::Path, n: usize) { + let path = dir.join("bt.idx"); + let _ = std::fs::remove_file(&path); + let mut bt = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER, 1024).unwrap(); + + bench("btree 插入(含 flush)", n, 2, || { + bt.clear().unwrap(); + for i in 0..n { + bt.insert(i as i64, i as u64).unwrap(); + } + bt.flush().unwrap(); + }); + + bench("btree 点查(全部命中)", n, 3, || { + for i in 0..n { + bt.get(i as i64).unwrap().expect("key must exist"); + } + }); + + bench("btree 全量遍历", n, 3, || { + let v = bt.iter_all().unwrap(); + assert_eq!(v.len(), n); + }); +} + +// 堆表 +/// 插行、全表扫描、按索引取行 +fn bench_heap(dir: &std::path::Path, n: usize) { + let path = dir.join("heap.db"); + let idx = dir.join("heap.idx"); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&idx); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, 1024).unwrap(); + + bench("heap 插入(只在缓冲池,不 flush)", n, 2, || { + t.replace_all(&[]).unwrap(); + for i in 0..n { + t.insert_row(&row(i as i64)).unwrap(); + } + }); + + bench("heap 插入 + flush", n, 2, || { + t.replace_all(&[]).unwrap(); + for i in 0..n { + t.insert_row(&row(i as i64)).unwrap(); + } + t.flush().unwrap(); + }); + + bench("heap 批量插入 + flush(一次落盘)", n, 2, || { + t.replace_all(&[]).unwrap(); + let rows: Vec = (0..n as i64).map(row).collect(); + t.insert_rows(&rows).unwrap(); + t.flush().unwrap(); + }); + + bench("heap 全表扫描", n, 3, || { + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), n); + }); +} + +// WAL +/// 追加写与清空 +fn bench_wal(dir: &std::path::Path, n: usize) { + let path = dir.join("wal.log"); + let _ = std::fs::remove_file(&path); + let wal = Wal::new(&path); + let op = WalOp::Insert { + table: "t".to_string(), + row: row(1), + }; + + bench_once("wal 追加(每条都 fsync)", n, || { + for _ in 0..n { + wal.append(&op).unwrap(); + } + }); + wal.clear().unwrap(); + + bench_once("wal 清空(每条都 fsync)", n, || { + for _ in 0..n { + wal.clear().unwrap(); + } + }); + wal.clear().unwrap(); + + bench_once("wal 截断(不落盘)", n, || { + for _ in 0..n { + wal.truncate().unwrap(); + } + }); + + // 拆分:一次 fsync 本身多贵,一次 open 多贵 + use std::fs::OpenOptions; + use std::io::Write; + // 大小和一条真实的 WAL 记录同量级(~200 字节) + let bytes = vec![0u8; 200]; + bench_once(" └ open + write + fsync", n, || { + for _ in 0..n { + let mut f = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .unwrap(); + f.write_all(&bytes).unwrap(); + f.sync_data().unwrap(); + } + }); + let mut open_file = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .unwrap(); + bench_once(" └ 复用句柄:write + fsync", n, || { + for _ in 0..n { + open_file.write_all(&bytes).unwrap(); + open_file.sync_data().unwrap(); + } + }); + bench_once(" └ only fsync(不开不写)", n, || { + for _ in 0..n { + open_file.sync_data().unwrap(); + } + }); + bench_once(" └ only open/close", n, || { + for _ in 0..n { + let _ = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .unwrap(); + } + }); + wal.clear().unwrap(); +} + +// 服务端写入路径 +/// 复刻 server.rs 里 write_via_wal 的顺序:清 WAL -> 写 WAL -> 改数据 -> flush -> 清 WAL +fn bench_write_path(dir: &std::path::Path, n: usize) { + let path = dir.join("wp.db"); + let idx = dir.join("wp.idx"); + let wal_path = dir.join("wp.wal"); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&idx); + let _ = std::fs::remove_file(&wal_path); + + let mut t = HeapTable::open_indexed( + &path, + &idx, + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + 1024, + ) + .unwrap(); + let wal = Wal::new(&wal_path); + + bench_once("写入路径 单条 INSERT(端到端)", n, || { + for i in 0..n as i64 { + wal.truncate().unwrap(); + let op = WalOp::Insert { + table: "t".to_string(), + row: row(i), + }; + wal.append(&op).unwrap(); + t.insert_row(&row(i)).unwrap(); + t.flush().unwrap(); + wal.clear().unwrap(); + } + }); + + // 拆开量,好定位时间花在哪一步 + bench_once(" └ wal.truncate()(不落盘)", n, || { + for _ in 0..n { + wal.truncate().unwrap(); + } + }); + + let op = WalOp::Insert { + table: "t".to_string(), + row: row(1), + }; + bench_once(" └ wal.append()(含 fsync)", n, || { + for _ in 0..n { + wal.append(&op).unwrap(); + } + }); + wal.clear().unwrap(); + + bench_once(" └ wal.clear()(含 fsync)", n, || { + for _ in 0..n { + wal.clear().unwrap(); + } + }); + + bench_once(" └ heap.insert_row()(含写索引)", n, || { + // 主循环已经插过 0..n 了,这里换个号段,免得撞上"索引列唯一" + for i in n as i64..2 * n as i64 { + t.insert_row(&row(i)).unwrap(); + } + }); + + bench_once(" └ heap.flush()", n, || { + for _ in 0..n { + t.flush().unwrap(); + } + }); +} + +fn main() { + let n: usize = std::env::var("CHUSQL_BENCH_N") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2000); + + println!("chusql-storage 基准:n = {},页大小 = {} 字节,btree order = {}", + n, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER); + println!("每项取最快一轮;带 fsync 的项只跑一次(跑多了太慢)\n"); + + let dir = tempfile::tempdir().unwrap(); + + println!("[页 IO]"); + bench_page(dir.path(), n); + + println!("\n[B+ 树]"); + bench_btree(dir.path(), n); + + println!("\n[堆表]"); + bench_heap(dir.path(), n); + + println!("\n[WAL]"); + bench_wal(dir.path(), n.min(200)); + + println!("\n[服务端写入路径]"); + bench_write_path(dir.path(), n.min(200)); +} diff --git a/benchmark/fullchain/chusql-fullchain.cabal b/benchmark/fullchain/chusql-fullchain.cabal new file mode 100644 index 0000000..67bf3d1 --- /dev/null +++ b/benchmark/fullchain/chusql-fullchain.cabal @@ -0,0 +1,17 @@ +cabal-version: 2.4 + +name: chusql-fullchain +version: 1.0.0 +synopsis: ChuSQL 全链路基准:Haskell 引擎 + Rust 存储进程一起跑 +build-type: Simple + +executable chusql-fullchain + main-is: Main.hs + hs-source-dirs: src + build-depends: base >= 4.14 && < 5 + , chusql-engine + , directory + , filepath + , process + default-language: Haskell2010 + ghc-options: -O2 -Wall diff --git a/benchmark/fullchain/src/Main.hs b/benchmark/fullchain/src/Main.hs new file mode 100644 index 0000000..b4ae703 --- /dev/null +++ b/benchmark/fullchain/src/Main.hs @@ -0,0 +1,435 @@ +module Main where + +import ChuSQL.Engine (runStatementM) +import ChuSQL.Model (Row, Value (..)) +import ChuSQL.Storage (IndexResult (..)) +import ChuSQL.Storage.IPC ( + IPCStorage (..), + Request (..), + Response (..), + TableInfo (..), + closeConnection, + doInsert, + doListCatalog, + doLookupByColumn, + sendRequest, + ) +import ChuSQL.Syntax.Parser (parseStatement) +import Control.Concurrent (threadDelay) +import Control.Exception (SomeException, bracket, evaluate, try) +import Control.Monad (filterM, unless, when) +import Data.List (intercalate, isInfixOf, isPrefixOf) +import GHC.Clock (getMonotonicTime) +import System.Directory ( + createDirectoryIfMissing, + doesFileExist, + getTemporaryDirectory, + removeDirectoryRecursive, + ) +import System.Environment (getArgs, getEnvironment, setEnv) +import System.Exit (exitFailure) +import System.FilePath (()) +import System.Process ( + CreateProcess (..), + ProcessHandle, + StdStream (..), + createProcess, + proc, + terminateProcess, + waitForProcess, + ) +import Text.Printf (printf) + +-- 全链路基准:Haskell 引擎和 Rust 存储进程**一起启动**,跑真实情景。 +-- +-- 一条语句真正走过的路: +-- SQL 文本 → 词法/语法分析 → 语义检查 → 关系代数 → 命名管道(JSON 行) +-- → Rust:WAL(fsync)→ 堆表/B+ 树 → 页缓冲池 → 落盘 +-- → 结果原路返回 +-- +-- 计时用**墙上时钟(单调时钟)**,不是 CPU 时间:这条链路大部分工夫花在等 IO +-- 和跨进程往返上,用 CPU 时间会把这些等待整个漏掉。 +-- +-- 跑法(先把 Rust 那边编译成 release): +-- cd chusql-storage && cargo build --release +-- cd benchmark/fullchain && stack run # 默认 users=orders=200 +-- stack run -- 500 500 # 换规模 +-- stack run -- 200 200 ../../chusql-storage/target/release/server.exe # 指定存储进程 + +-- * 命令行与存储进程 + +-- | 找 Rust 存储进程:优先 release,其次 debug +findServer :: FilePath -> IO FilePath +findServer "" = do + let dirs = + [ "chusql-storage/target" + , "../../chusql-storage/target" + , "../chusql-storage/target" + , "target" + ] + modes = ["release", "debug"] + exes = ["server.exe", "server"] + candidates = [d m e | d <- dirs, m <- modes, e <- exes] + found <- filterM doesFileExist candidates + case found of + (p : _) -> do + unless ("release" `isInfixOf` p) $ + putStrLn "!! 找到的是 debug 版存储进程,数字会偏慢;建议 cargo build --release" + pure p + [] -> do + putStrLn "找不到 Rust 存储进程。先编译:" + putStrLn " cd chusql-storage && cargo build --release" + exitFailure +findServer p = do + ok <- doesFileExist p + if ok + then pure p + else do + putStrLn ("指定的存储进程不存在:" ++ p) + exitFailure + +-- | 子进程环境:继承当前环境,但把 CHUSQL_* 换成本次专用的 +childEnv :: [(String, String)] -> IO [(String, String)] +childEnv extra = do + base <- getEnvironment + let stripped = filter (\(k, _) -> not ("CHUSQL_" `isPrefixOf` k)) base + pure (stripped ++ extra) + +-- | 起存储进程并等管道可连 +startServer :: FilePath -> String -> FilePath -> IO ProcessHandle +startServer serverPath pipe dataDir = do + envs <- + childEnv + [ ("CHUSQL_PIPE", pipe) + , ("CHUSQL_DATA_DIR", dataDir) + , ("CHUSQL_PAGE_SIZE", "4096") + , ("CHUSQL_BTREE_ORDER", "4") + , ("CHUSQL_LOG", "error") + ] + (_, _, _, ph) <- + createProcess + (proc serverPath []) {env = Just envs, std_out = NoStream, std_err = Inherit} + ready <- waitReady (200 :: Int) + unless ready $ do + putStrLn "存储进程 5 秒内没就绪" + exitFailure + pure ph + +-- | 轮询 ping,直到对面应答 +waitReady :: Int -> IO Bool +waitReady 0 = pure False +waitReady n = do + r <- try (sendRequest ReqPing) :: IO (Either SomeException Response) + case r of + Right RespPong -> pure True + _ -> threadDelay 25000 >> waitReady (n - 1) + +-- | 停存储进程并清掉缓存连接 +stopServer :: ProcessHandle -> FilePath -> IO () +stopServer ph dataDir = do + closeConnection + terminateProcess ph + _ <- waitForProcess ph + _ <- try (removeDirectoryRecursive dataDir) :: IO (Either SomeException ()) + pure () + +-- * 计时 + +-- | 量一次墙上耗时,并把结果行数强制出来(Haskell 是惰性的,不强制就等于没测) +timedRows :: IO (Either String [Row]) -> IO (Double, Either String Int) +timedRows act = do + t0 <- getMonotonicTime + r <- act + n <- case r of + Left _ -> pure (-1) + Right rows -> evaluate (length rows) + t1 <- getMonotonicTime + pure (t1 - t0, either Left (const (Right n)) r) + +-- | 跑一条 SQL 文本(整条链路) +runSql :: String -> IO (Either String [Row]) +runSql sql = case parseStatement sql of + Left e -> pure (Left e) + Right stmt -> runIPCStorage (runStatementM stmt) + +-- | 打印一行结果 +report :: String -> Either String Int -> Double -> IO () +report label result dt = case result of + Left e -> printf "%-38s %10s !! %s\n" label "失败" e + Right n -> printf "%-38s %8.1f ms %8d 行\n" label (dt * 1000) n + +-- | 单独量一条 SQL(先预热一遍):只适合可以重复跑的语句(SELECT) +benchSql :: String -> String -> IO (Either String Int) +benchSql label sql = do + _ <- runSql sql -- 预热:连接、页缓存、优化器都先热一次 + (dt, r) <- timedRows (runSql sql) + report label r dt + pure r + +-- | 只跑一次的语句(CREATE TABLE 这类重跑会报错的) +benchSqlOnce :: String -> String -> IO (Either String Int) +benchSqlOnce label sql = do + (dt, r) <- timedRows (runSql sql) + report label r dt + pure r + +-- * 场景 + +-- | 造一条 INSERT 的 SQL 文本 +userSql :: Int -> String +userSql i = + printf "INSERT INTO users (id, name, age) VALUES (%d, 'user%d', %d)" i i (i `mod` 100) + +-- | 造一条订单的 INSERT +orderSql :: Int -> Int -> String +orderSql u j = + printf + "INSERT INTO orders (id, user_id, product) VALUES (%d, %d, 'item%d')" + j + (1 + (j - 1) `mod` u) + (j `mod` 50) + +-- | 批量插入并报平均耗时 +batchInsert :: String -> Int -> (Int -> String) -> IO (Either String ()) +batchInsert label n mkSql = do + t0 <- getMonotonicTime + results <- mapM (runSql . mkSql) [1 .. n] + t1 <- getMonotonicTime + let total = t1 - t0 + errs = [e | Left e <- results] + case errs of + [] -> do + printf + "%-38s %8.1f ms %7.3f ms/条\n" + label + (total * 1000) + (total * 1000 / fromIntegral n) + pure (Right ()) + (e : _) -> do + printf "%-38s %10s !! %s\n" label "失败" e + pure (Left e) + +-- | 一次点查(原始 IPC,不经过引擎) +benchLookup :: String -> String -> Int -> Int -> IO () +benchLookup table column key times = do + _ <- doLookupByColumn table column key + t0 <- getMonotonicTime + results <- mapM (const (doLookupByColumn table column key)) [1 .. times] + t1 <- getMonotonicTime + let total = t1 - t0 + hit = length [() | Right (IndexRow (Just _)) <- results] + printf + "%-38s %8.1f ms %7.3f ms/次(命中 %d/%d)\n" + "lookup_by_index(原始 IPC,按 id)" + (total * 1000) + (total * 1000 / fromIntegral times) + hit + times + +-- | 量 list_catalog:现在每条语句都要先来一发(只取结构,不取行) +benchListCatalog :: Int -> IO () +benchListCatalog n = do + t0 <- getMonotonicTime + _ <- mapM (const doListCatalog) [1 .. n] + t1 <- getMonotonicTime + let total = t1 - t0 + printf + "%-38s %8.1f ms %7.3f ms/次\n" + "list_catalog(取结构)" + (total * 1000) + (total * 1000 / fromIntegral n) + +-- | 量原始 IPC 插入:不经过引擎,把"协议 + 存储"和"引擎"分开 +benchRawInsert :: String -> [Row] -> IO () +benchRawInsert table rows = do + t0 <- getMonotonicTime + results <- mapM (\r -> doInsert table r) rows + t1 <- getMonotonicTime + let total = t1 - t0 + n = length rows + errs = [e | Left e <- results] + case errs of + [] -> + printf + "%-38s %8.1f ms %7.3f ms/条\n" + ("原始 IPC insert × " ++ show n) + (total * 1000) + (total * 1000 / fromIntegral n) + (e : _) -> printf "%-38s %10s !! %s\n" "原始 IPC insert" "失败" e + +-- | 造一行 users 数据(给原始 IPC 用) +userRow :: Int -> Row +userRow i = [("id", VInt i), ("name", VStr ("user" ++ show i)), ("age", VInt (i `mod` 100))] + +-- | 依次跑一批语句,返回 (耗时秒, 错误清单) +runBatch :: [String] -> IO (Double, [String]) +runBatch sqls = do + t0 <- getMonotonicTime + results <- mapM runSql sqls + t1 <- getMonotonicTime + pure (t1 - t0, [e | Left e <- results]) + +-- | 把 1..n 按 size 切成一段段 +chunked :: Int -> Int -> [[Int]] +chunked size n = go [1 .. n] + where + -- \| 每次切下 size 个 + go [] = [] + go xs = let (a, b) = splitAt size xs in a : go b + +-- | 一条带多行的 INSERT(一条语句 N 行,只落一次盘) +multiInsertSql :: [Int] -> String +multiInsertSql ids = + "INSERT INTO batch_t (id, name, age) VALUES " + ++ intercalate ", " [printf "(%d, 'b%d', %d)" i i (i `mod` 100) | i <- ids] + +-- | 打印一张表的统计 +printStats :: TableInfo -> IO () +printStats i = + printf + " %-12s %6d 行 索引=%s\n 统计=%s\n" + (tiTable i) + (tiRows i) + (if null (tiIndexes i) then "无" else unwords (tiIndexes i)) + ( unwords + [ n ++ ":" ++ show d ++ (if c then "+(下界)" else "") + | (n, d, c) <- tiStats i + ] + ) + +-- | 主场景 +runScenarios :: Int -> Int -> IO () +runScenarios u o = do + putStrLn "\n[建表]" + r1 <- benchSqlOnce "CREATE TABLE users" "CREATE TABLE users (id int, name str, age int)" + r2 <- benchSqlOnce "CREATE TABLE orders" "CREATE TABLE orders (id int, user_id int, product str)" + case (r1, r2) of + (Right _, Right _) -> pure () + _ -> do + putStrLn "建表失败,后面不用跑了" + exitFailure + + putStrLn "\n[写入:每条 INSERT 都是一次完整落盘]" + _ <- batchInsert (printf "%d 条 INSERT INTO users" u) u userSql + _ <- batchInsert (printf "%d 条 INSERT INTO orders" o) o (orderSql u) + + putStrLn "\n[分解:同一条 INSERT 拆成两半,都用原始 IPC,不经过引擎]" + _ <- benchSqlOnce "CREATE TABLE raw_users" "CREATE TABLE raw_users (id int, name str, age int)" + benchListCatalog 200 + benchRawInsert "raw_users" [userRow i | i <- [1 .. u]] + + putStrLn "\n[读取与查询:整条语句,含引擎侧的解析与语义检查]" + _ <- benchSql "SELECT * FROM users" "SELECT * FROM users" + ageR <- benchSql "SELECT name FROM users WHERE age > 90" "SELECT name FROM users WHERE age > 90" + + putStrLn "\n[读取与查询:连接]" + joinR <- + benchSql + "JOIN + WHERE(哈希连接)" + "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 90" + + putStrLn "\n[点查]" + benchLookup "users" "id" (u `div` 2) 100 + _ <- benchSql "SELECT ... WHERE id = k(走 Lookup)" (printf "SELECT name FROM users WHERE id = %d" (u `div` 2)) + + putStrLn "\n[批量插入:一条语句带多行,N 行只落一次盘]" + _ <- benchSqlOnce "CREATE TABLE batch_t" "CREATE TABLE batch_t (id int, name str, age int)" + let groups = chunked 100 u + (bt, batchErrs) <- runBatch (map multiInsertSql groups) + case batchErrs of + [] -> + printf + "%-38s %8.1f ms %7.3f ms/行(%d 行/条 × %d 条语句)\n" + "多行 INSERT" + (bt * 1000) + (bt * 1000 / fromIntegral u) + (100 :: Int) + (length groups) + (e : _) -> printf "%-38s %10s !! %s\n" "多行 INSERT" "失败" e + + putStrLn "\n[行级删除:代价和删几行无关,只和扫描 + 一次落盘有关]" + _ <- benchSqlOnce "DELETE FROM users WHERE id = k(1 行)" (printf "DELETE FROM users WHERE id = %d" (u `div` 2)) + _ <- benchSqlOnce "DELETE FROM users(全部)" "DELETE FROM users" + + putStrLn "\n[二级索引:没有索引回退扫描,建完索引走索引]" + _ <- benchSqlOnce "CREATE TABLE items" "CREATE TABLE items (id int, code int)" + let itemGroups = chunked 100 o + itemSql ids = + "INSERT INTO items (id, code) VALUES " + ++ intercalate ", " [printf "(%d, %d)" i (i * 7 + 1) | i <- ids] + (it, itemErrs) <- runBatch (map itemSql itemGroups) + case itemErrs of + [] -> pure () + (e : _) -> printf " !! %s\n" e + printf " (先把 %d 行灌进 items:%.1f ms)\n" o (it * 1000) + let targetCode = (o `div` 2) * 7 + 1 + _ <- + benchSql + "SELECT ... WHERE code = k(无索引)" + (printf "SELECT id FROM items WHERE code = %d" targetCode) + _ <- benchSqlOnce "CREATE INDEX ON items (code)" "CREATE INDEX ON items (code)" + _ <- + benchSql + "SELECT ... WHERE code = k(走索引)" + (printf "SELECT id FROM items WHERE code = %d" targetCode) + + putStrLn "\n[数据字典统计]" + cat <- doListCatalog + case cat of + Left e -> printf " 取统计失败:%s\n" e + Right infos -> mapM_ printStats infos + + putStrLn "\n[校验]" + let expectAge = 9 * (u `div` 100) + max 0 (u `mod` 100 - 90) + printf + "age > 90 命中 %s(按数据分布应为 %d 行)\n" + (either show show ageR) + expectAge + when (u == o) $ + printf + "JOIN 命中 %s(每个用户恰好 1 张订单,应与上一行相同 = %d)\n" + (either show show joinR) + expectAge + unless (u == o) $ + putStrLn "(users 与 orders 行数不同,JOIN 行数没有简单公式,只打印不校验)" + printf "多行 INSERT 灌进去 %d 行(应为 %d)\n" u u + printf "二级索引那一行:code = %d 应该查到 id = %d\n" targetCode (o `div` 2) + +-- | 主入口 +main :: IO () +main = do + args <- getArgs + let pick i d = if length args > i then args !! i else d + u = read (pick 0 "200") :: Int + o = read (pick 1 "200") :: Int + serverArg = pick 2 "" + serverPath <- findServer serverArg + + -- 每次跑用独立的管道名与数据目录,避免和别的进程抢 + tmp <- getTemporaryDirectory + stamp <- getMonotonicTime + let pipe = "chusql-fullchain-" ++ show (round (stamp * 1e6) :: Int) + dataDir = tmp pipe + createDirectoryIfMissing True dataDir + setEnv "CHUSQL_PIPE" pipe + + printf "全链路基准:Haskell 引擎 ←命名管道→ Rust 存储进程\n" + printf "存储进程:%s\n" serverPath + printf "管道:%s\n" pipe + printf "数据目录:%s\n" dataDir + printf "数据规模:users = %d 行,orders = %d 行\n" u o + printf "计时:墙上时钟(单调);每条语句都走完整链路\n" + + bracket + (do + t1 <- getMonotonicTime + ph <- startServer serverPath pipe dataDir + t2 <- getMonotonicTime + printf "\n[启动] 存储进程就绪,用时 %.2f s\n" (t2 - t1) + pure ph + ) + (\ph -> do + putStrLn "\n[收尾] 停存储进程、清理数据目录" + stopServer ph dataDir + ) + (const (runScenarios u o)) diff --git a/benchmark/fullchain/stack.yaml b/benchmark/fullchain/stack.yaml new file mode 100644 index 0000000..eacce59 --- /dev/null +++ b/benchmark/fullchain/stack.yaml @@ -0,0 +1,5 @@ +resolver: lts-24.59 + +packages: + - . + - ../../chusql-engine diff --git a/benchmark/fullchain/stack.yaml.lock b/benchmark/fullchain/stack.yaml.lock new file mode 100644 index 0000000..e3f9041 --- /dev/null +++ b/benchmark/fullchain/stack.yaml.lock @@ -0,0 +1,12 @@ +# This file was autogenerated by Stack. +# You should not edit this file by hand. +# For more information, please see the documentation at: +# https://docs.haskellstack.org/en/stable/topics/lock_files + +packages: [] +snapshots: +- completed: + sha256: 0f728e30e843d7f00460ad56858b995a3b227fca683659c7df9f2fc3988a3ce1 + size: 732465 + url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/24/59.yaml + original: lts-24.59 diff --git a/chusql-engine/chusql-engine.cabal b/chusql-engine/chusql-engine.cabal index f1f839c..9d4a8a9 100644 --- a/chusql-engine/chusql-engine.cabal +++ b/chusql-engine/chusql-engine.cabal @@ -26,10 +26,12 @@ library build-depends: base >= 4.14 && < 5 , aeson , bytestring + , hashable , megaparsec , parser-combinators , scientific , text + , unordered-containers default-language: Haskell2010 ghc-options: -Wall diff --git a/chusql-engine/src/ChuSQL/Algebra/Eval.hs b/chusql-engine/src/ChuSQL/Algebra/Eval.hs index 9db8993..6221f53 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Eval.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Eval.hs @@ -4,9 +4,10 @@ import ChuSQL.Algebra.Expr (evalCondForRow) import ChuSQL.Algebra.Op import ChuSQL.Algebra.Sort (sortRows) import ChuSQL.Model -import ChuSQL.Storage (MonadStorage (..)) +import ChuSQL.Storage (IndexResult (..), MonadStorage (..)) import ChuSQL.Syntax.AST (Expr (..)) import Control.Monad (filterM, foldM) +import qualified Data.HashMap.Strict as HM -- 执行:按算子树算出结果行(Scan 阶段会加别名前缀)。 @@ -18,19 +19,55 @@ project ["*"] row = row project cs row = [(c, v) | (c, v) <- row, c `elem` cs] -- | 给列名加别名前缀 -addPrefix :: String -> Row -> Row -addPrefix prefix = map (\(k, v) -> (prefix ++ k, v)) - --- | 主键匹配条件(id = k) -keyCondition :: Int -> Expr -keyCondition k = Eq (Col "id") (LitInt k) +addPrefix :: Maybe String -> Row -> Row +addPrefix mAlias = map (\(k, v) -> (qualify mAlias k, v)) -- | 扫描一张表 evalScan :: Database -> Maybe String -> String -> Either String [Row] evalScan db mAlias tbl = do table <- lookupTable db tbl - let prefix = maybe "" (++ ".") mAlias - Right (map (addPrefix prefix) (tableRows table)) + Right (map (addPrefix mAlias) (tableRows table)) + +-- | 点查退化成"扫描 + 按条件筛"时用的条件:`列 = 值`(列名带上别名前缀) +lookupCondition :: Maybe String -> String -> Int -> Expr +lookupCondition mAlias col k = Eq (Col (qualify mAlias col)) (LitInt k) + +-- * 等值连接 + +-- | 等值连接的两侧键:左行取哪列、右行取哪列 +data EquiKeys = EquiKeys + { ekLeft :: String + , ekRight :: String + } + +-- | 一批行的列名(空批给空清单,此时一律退回嵌套循环) +rowsCols :: [Row] -> [String] +rowsCols (r : _) = map fst r +rowsCols [] = [] + +-- | 认出 a = b 这种两列等值条件,并且能分清谁在左谁在右 +equiKeys :: [Row] -> [Row] -> Expr -> Maybe EquiKeys +equiKeys lrows rrows (Eq (Col a) (Col b)) + | a /= b, a `elem` lcols, b `elem` rcols = Just (EquiKeys a b) + | a /= b, b `elem` lcols, a `elem` rcols = Just (EquiKeys b a) + where + lcols = rowsCols lrows + rcols = rowsCols rrows +equiKeys _ _ _ = Nothing + +-- | 哈希连接:右表建哈希表,左表逐行探测。 +-- 输出顺序与嵌套循环一致(左表顺序为主,右表顺序为辅)。 +hashJoin :: EquiKeys -> [Row] -> [Row] -> [Row] +hashJoin keys lrows rrows = concatMap probe lrows + where + -- 同一键的右行按原顺序排在桶里 + buckets = + HM.fromListWith + (flip (++)) + [(v, [r]) | r <- rrows, Just v <- [lookup (ekRight keys) r]] + probe l = case lookup (ekLeft keys) l of + Nothing -> [] + Just v -> [l ++ r | r <- HM.lookupDefault [] v buckets] -- | 把两边的行按条件配对(边配边筛,不先建整张积表) filterPairs :: (Row -> Either String Bool) -> [Row] -> [Row] -> Either String [Row] @@ -45,14 +82,21 @@ filterPairs cond lrows rrows = go lrows [] ok <- cond row pure (if ok then row : acc else acc) +-- | 一次连接:能认出等值键就走哈希,否则退回嵌套循环 +joinRows :: Expr -> [Row] -> [Row] -> Either String [Row] +joinRows cond lrows rrows = case equiKeys lrows rrows cond of + Just keys -> Right (hashJoin keys lrows rrows) + Nothing -> filterPairs (evalCondForRow cond) lrows rrows + -- * 求值 -- | 纯求值(不需要存储) evalRelOp :: Database -> RelOp -> Either String [Row] evalRelOp db (Scan mAlias tbl) = evalScan db mAlias tbl -evalRelOp db (Lookup tbl k) = do - rows <- evalScan db Nothing tbl - filterM (evalCondForRow (keyCondition k)) rows +-- 纯求值没有索引可问:退回"扫描 + 按条件筛",和未优化时的 Filter 完全等价 +evalRelOp db (Lookup mAlias tbl col k) = do + rows <- evalScan db mAlias tbl + filterM (evalCondForRow (lookupCondition mAlias col k)) rows evalRelOp db (Filter e op) = do rows <- evalRelOp db op filterM (evalCondForRow e) rows @@ -68,9 +112,9 @@ evalRelOp db (Limit n op) = do evalRelOp db (Join left right cond) = do lrows <- evalRelOp db left rrows <- evalRelOp db right - filterPairs (evalCondForRow cond) lrows rrows + joinRows cond lrows rrows --- | 单子求值:Lookup 问存储 +-- | 单子求值:Lookup 先问存储有没有索引,没有就退回扫描 evalRelOpM :: (MonadStorage m) => RelOp -> m (Either String [Row]) evalRelOpM (Scan mAlias t) = do result <- scan t @@ -78,13 +122,19 @@ evalRelOpM (Scan mAlias t) = do rows <- result case mAlias of Nothing -> Right rows - Just a -> Right (map (addPrefix (a ++ ".")) rows) -evalRelOpM (Lookup t k) = do - result <- lookupByKey t k - pure $ case result of - Left e -> Left e - Right Nothing -> Right [] - Right (Just row) -> Right [row] + Just _ -> Right (map (addPrefix mAlias) rows) +evalRelOpM (Lookup mAlias t col k) = do + result <- lookupByColumn t col k + case result of + Left e -> pure (Left e) + -- 这个列上没有索引:老老实实扫一遍(结果和未优化时一样) + Right NoIndex -> do + rows <- evalRelOpM (Scan mAlias t) + pure $ do + rs <- rows + filterM (evalCondForRow (lookupCondition mAlias col k)) rs + Right (IndexRow Nothing) -> pure (Right []) + Right (IndexRow (Just row)) -> pure (Right [addPrefix mAlias row]) evalRelOpM (Filter p x) = do result <- evalRelOpM x pure $ do @@ -105,4 +155,4 @@ evalRelOpM (Join l r cond) = do pure $ do left <- ls right <- rs - filterPairs (evalCondForRow cond) left right + joinRows cond left right diff --git a/chusql-engine/src/ChuSQL/Algebra/Op.hs b/chusql-engine/src/ChuSQL/Algebra/Op.hs index b5bd289..0f1602a 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Op.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Op.hs @@ -7,7 +7,11 @@ import ChuSQL.Syntax.AST (Expr, SortDir) -- | 算子树的一个节点 data RelOp = Scan (Maybe String) String - | Lookup String Int + | -- \| 按某一列的值点查一行:别名、表、列、值。 + -- 只是"这里可以走点查"的意思,到底有没有索引由存储层回答 + -- (那个列上没有索引就退回全表扫描,见 Eval)。 + -- 别名要留着,回来的一行才能和 `Scan 别名 表` 长得一样。 + Lookup (Maybe String) String String Int | Filter Expr RelOp | Project [String] RelOp | Sort [(String, SortDir)] RelOp @@ -21,7 +25,7 @@ renderPlan = go 0 where go ind op = case op of Scan a t -> pad ++ "Scan " ++ show a ++ " " ++ show t - Lookup t k -> pad ++ "Lookup " ++ show t ++ " " ++ show k + Lookup _ t c k -> pad ++ "Lookup " ++ show t ++ "." ++ show c ++ " = " ++ show k Filter e x -> pad ++ "Filter " ++ show e ++ "\n" ++ go (ind + 1) x Project c x -> pad ++ "Project " ++ show c ++ "\n" ++ go (ind + 1) x Sort s x -> pad ++ "Sort " ++ show s ++ "\n" ++ go (ind + 1) x diff --git a/chusql-engine/src/ChuSQL/Algebra/Optimize.hs b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs index 9033b36..bd9f1df 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Optimize.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs @@ -16,7 +16,7 @@ relOpCols db (Scan mAlias tbl) = Just t -> map (prefix ++) (colNames t) where prefix = maybe "" (++ ".") mAlias -relOpCols db (Lookup tbl _) = relOpCols db (Scan Nothing tbl) +relOpCols db (Lookup mAlias tbl _ _) = relOpCols db (Scan mAlias tbl) relOpCols db (Filter _ x) = relOpCols db x relOpCols _ (Project ["*"] _) = ["*"] relOpCols _ (Project cols _) = cols @@ -104,7 +104,7 @@ pushProject db need (Join l r c) rNeed = if needsAll rCols then ["*"] else [x | x <- tot, x `elem` rCols] in Join (pushProject db lNeed l) (pushProject db rNeed r) c pushProject db need leaf@(Scan _ _) = pushProjectLeaf db need leaf -pushProject db need leaf@(Lookup _ _) = pushProjectLeaf db need leaf +pushProject db need leaf@(Lookup _ _ _ _) = pushProjectLeaf db need leaf -- * 常量折叠 @@ -131,8 +131,12 @@ foldConstants e = e -- * 执行优化 -- | 单节点重写 rewriteNode :: Database -> RelOp -> RelOp -rewriteNode _ (Filter (Eq (Col k) (LitInt v)) (Scan _ t)) - | k == "id" = Lookup t v +-- 单列等值条件改成点查:这里是"可以走索引"的意思, +-- 到底走不走得成由存储层回答(这个列上没有索引就退回全表扫描,见 Eval), +-- 所以优化器不必先知道表上到底有哪些索引。 +-- 别名要留在节点里:回来的一行必须和 `Scan 别名 表` 长得一样,不然后面取列就对不上了。 +rewriteNode _ (Filter (Eq (Col k) (LitInt v)) (Scan mAlias t)) = + Lookup mAlias t (unqualify mAlias k) v rewriteNode db (Filter p (Join l r c)) = pushJoin db p l r c rewriteNode db (Filter p x) = case foldConstants p of LitBool True -> x diff --git a/chusql-engine/src/ChuSQL/Engine.hs b/chusql-engine/src/ChuSQL/Engine.hs index 892fab5..c5c25f1 100644 --- a/chusql-engine/src/ChuSQL/Engine.hs +++ b/chusql-engine/src/ChuSQL/Engine.hs @@ -9,7 +9,6 @@ import ChuSQL.Semantic (check) import ChuSQL.Storage (MonadStorage (..)) import ChuSQL.Storage.Memory (MemoryStorage (runMemoryStorage)) import ChuSQL.Syntax.AST -import Control.Monad (filterM) -- 引擎入口:语义检查 + 分发执行,以及内存实现与泛型版本。 @@ -29,7 +28,9 @@ rowsOf = fmap snd -- | 泛型入口:先检查再执行 runStatementM :: (MonadStorage m) => Statement -> m (Either String [Row]) runStatementM q = do - db <- snapshot + -- 只要结构:语义检查和查询优化都只看列,一行数据都不碰, + -- 所以这里不用取整库快照(那会把所有行拉一遍) + db <- schema case check db q of Left err -> pure (Left err) Right _ -> runStatementUncheckedM db q @@ -42,36 +43,43 @@ runStatementUncheckedM db q@Select{} = case translate q of Left e -> pure (Left e) Right relOp -> evalRelOpM (optimize db relOp) -runStatementUncheckedM _ (Insert tbl cols vals) = - case mapM (\e -> evalExpr e []) vals of +runStatementUncheckedM _ (Insert tbl cols rows) = + -- 多行一次交给存储:N 行只落一次盘(键怎么算、索引怎么写是存储层的事) + case mapM toRow rows of Left err -> pure (Left err) - Right values - | length cols /= length values -> - pure (Left "column count does not match value count") - | otherwise -> do - let row = zip cols values - key = case lookup "id" row of - Just (VInt k) -> Just k - _ -> Nothing - result <- insert tbl row key - pure (result >> Right []) + Right rs -> do + result <- insertMany tbl rs + pure (result >> Right []) + where + -- \| 一行字面量求值成一行数据 + toRow :: [Expr] -> Either String Row + toRow vals = do + values <- mapM (\e -> evalExpr e []) vals + if length cols /= length values + then Left "column count does not match value count" + else Right (zip cols values) runStatementUncheckedM _ (Delete tbl mWhere) = do rowsResult <- scan tbl case rowsResult of Left err -> pure (Left err) - Right rows -> do - let kept = case mWhere of - Nothing -> Right [] - Just e -> filterM (shouldKeep e) rows - case kept of - Left err -> pure (Left err) - Right keptRows -> do - result <- replaceAll tbl keptRows - pure (result >> Right []) + Right rows -> case splitByCondition mWhere rows of + Left err -> pure (Left err) + Right (doomed, kept) -> + -- 要删的行都有 id 就走行级删(存储层原地删,代价只和删几行有关); + -- 没有 id 的表定位不到行,只能整表写回。 + case mapM rowId doomed of + Right ids -> do + result <- deleteKeys tbl ids + pure (result >> Right []) + Left _ -> do + result <- replaceAll tbl kept + pure (result >> Right []) where - -- \| 保留 = 条件不成立 - shouldKeep :: Expr -> Row -> Either String Bool - shouldKeep e row = not <$> evalCondForRow e row + -- \| 这一行的 id(要拿来当删除键) + rowId :: Row -> Either String Int + rowId r = case lookup "id" r of + Just (VInt k) -> Right k + _ -> Left "row has no integer id" runStatementUncheckedM _ (Update tbl assigns mWhere) = do rowsResult <- scan tbl case rowsResult of @@ -97,9 +105,27 @@ runStatementUncheckedM _ (CreateTable name cols) = do runStatementUncheckedM _ (DropTable name) = do result <- dropTable name pure (result >> Right []) +runStatementUncheckedM _ (CreateIndex tbl col) = do + result <- createIndex tbl col + pure (result >> Right []) +runStatementUncheckedM _ (DropIndex tbl col) = do + result <- dropIndex tbl col + pure (result >> Right []) -- * 更新辅助 +-- | 按条件把行分成"要删的"和"留着的"两拨(没有条件就全都要删) +splitByCondition :: Maybe Expr -> [Row] -> Either String ([Row], [Row]) +splitByCondition cond = go [] [] + where + -- \| 一行一行过,保持原顺序;条件求值出错就立刻停(和 filterM 一样) + go doomed kept [] = Right (reverse doomed, reverse kept) + go doomed kept (r : rs) = do + hit <- case cond of + Nothing -> Right True + Just e -> evalCondForRow e r + go (if hit then r : doomed else doomed) (if hit then kept else r : kept) rs + -- | 依次求值并覆盖列 applyUpdates :: [(String, Expr)] -> Row -> Either String Row applyUpdates [] row = Right row diff --git a/chusql-engine/src/ChuSQL/Model.hs b/chusql-engine/src/ChuSQL/Model.hs index 6f6de07..e1f7d59 100644 --- a/chusql-engine/src/ChuSQL/Model.hs +++ b/chusql-engine/src/ChuSQL/Model.hs @@ -1,4 +1,8 @@ -module ChuSQL.Model (Column (..), Value (..), Row, Table (..), Database, colNames, colType, lookupTable, allColumns) where +module ChuSQL.Model (Column (..), Value (..), Row, Table (..), Database, colNames, colType, lookupTable, allColumns, qualify, unqualify) where + +import Data.Hashable (Hashable (..)) +import Data.List (stripPrefix) +import Data.Maybe (fromMaybe) -- 数据模型:列类型、值、行、表、数据库,外加几个查表小工具。 @@ -17,6 +21,12 @@ data Value | VBool Bool deriving (Show, Eq) +-- | 值要能当哈希键用(等值连接建哈希表) +instance Hashable Value where + hashWithSalt s (VInt n) = hashWithSalt (hashWithSalt s (0 :: Int)) n + hashWithSalt s (VStr t) = hashWithSalt (hashWithSalt s (1 :: Int)) t + hashWithSalt s (VBool b) = hashWithSalt (hashWithSalt s (2 :: Int)) b + -- | 一行:列名 -> 值 type Row = [(String, Value)] @@ -47,3 +57,13 @@ lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db) -- | 投影哨兵 "*" allColumns :: String allColumns = "*" + +-- | 给列名加别名前缀:`Just "u"` + `name` -> `u.name`(没有别名就原样) +qualify :: Maybe String -> String -> String +qualify Nothing c = c +qualify (Just a) c = a ++ "." ++ c + +-- | 去掉别名前缀:`Just "u"` + `u.name` -> `name`(没有这个前缀就原样返回) +unqualify :: Maybe String -> String -> String +unqualify Nothing c = c +unqualify (Just a) c = fromMaybe c (stripPrefix (a ++ ".") c) diff --git a/chusql-engine/src/ChuSQL/Semantic.hs b/chusql-engine/src/ChuSQL/Semantic.hs index e7cd7fd..bc2b097 100644 --- a/chusql-engine/src/ChuSQL/Semantic.hs +++ b/chusql-engine/src/ChuSQL/Semantic.hs @@ -114,10 +114,12 @@ check db q = case q of checkColumns "SELECT" env cols checkColumns "ORDER BY" env (map fst orderBy) mapM_ (checkBool "WHERE" env) mWhere - Insert tbl cols vals -> do + Insert tbl cols rows -> do t <- lookupTable db tbl checkColumns "INSERT" (tableCols t) cols - mapM_ (checkValue t) (zip cols vals) + if null rows + then Left "INSERT: no values" + else mapM_ (mapM_ (checkValue t) . zip cols) rows Delete tbl mWhere -> do t <- lookupTable db tbl mapM_ (checkBool "WHERE" (tableCols t)) mWhere @@ -138,3 +140,16 @@ check db q = case q of if null name then Left "DROP TABLE: empty table name" else Right () + CreateIndex tbl col -> do + t <- lookupTable db tbl + if col `elem` tableCols' t + then Right () + else Left ("CREATE INDEX: unknown column: " ++ col) + DropIndex tbl col -> do + t <- lookupTable db tbl + if col `elem` tableCols' t + then Right () + else Left ("DROP INDEX: unknown column: " ++ col) + where + -- \| 表的列名清单 + tableCols' = map fst . tableCols diff --git a/chusql-engine/src/ChuSQL/Storage.hs b/chusql-engine/src/ChuSQL/Storage.hs index b06a0e5..0530e7d 100644 --- a/chusql-engine/src/ChuSQL/Storage.hs +++ b/chusql-engine/src/ChuSQL/Storage.hs @@ -1,24 +1,66 @@ module ChuSQL.Storage ( MonadStorage (..), + IndexResult (..), ) where -import ChuSQL.Model (Column, Database, Row) +import ChuSQL.Model (Column, Database, Row, Table (..), Value (..)) -- 存储层抽象:上层只认 domain 类型,不关心底层实现。 --- | 存储接口:读写表、按主键取行、取快照 +-- | 一次索引点查的结果 +data IndexResult + = -- \| 有索引,查过了:Just 命中一行,Nothing 没这行 + IndexRow (Maybe Row) + | -- \| 没有这个索引 —— 调用方应该退回全表扫描 + NoIndex + deriving (Show, Eq) + +-- | 存储接口:读写表、按索引取行、取快照 class (Monad m) => MonadStorage m where -- \| 全表扫描 scan :: String -> m (Either String [Row]) - -- \| 追加一行 - insert :: String -> Row -> Maybe Int -> m (Either String ()) + -- \| 追加一行。 + -- 要写哪些索引由**存储层按自己的元数据**决定(表上有哪些索引它自己清楚), + -- 调用方不需要指定 key。 + insert :: String -> Row -> m (Either String ()) + + -- \| 一批行一起追加。 + -- 默认逐行调 `insert`;能批量的实现(比如走 WAL 的 IPC 存储)会覆盖它, + -- 好处是 N 行只落一次盘。 + insertMany :: String -> [Row] -> m (Either String ()) + insertMany t = go + where + -- \| 一行一行来,遇到错就停 + go [] = pure (Right ()) + go (r : rs) = do + x <- insert t r + case x of + Left e -> pure (Left e) + Right _ -> go rs + + -- \| 按 `id` 批量删行。 + -- 默认实现是"全读出来、筛掉、整表写回"(对内存实现足够); + -- 真正能在原地删的实现会覆盖它。 + deleteKeys :: String -> [Int] -> m (Either String ()) + deleteKeys t ks = do + rows <- scan t + case rows of + Left e -> pure (Left e) + Right rs -> replaceAll t [r | r <- rs, not (doomed r)] + where + -- \| 这一行的 id 在待删清单里吗 + doomed r = case lookup "id" r of + Just (VInt k) -> k `elem` ks + _ -> False -- \| 整表替换 replaceAll :: String -> [Row] -> m (Either String ()) - -- \| 按主键取一行 - lookupByKey :: String -> Int -> m (Either String (Maybe Row)) + -- \| 按某一列的索引取一行。 + -- 默认回答"没有索引"(内存实现不需要索引),于是上层会自己扫一遍。 + lookupByColumn :: String -> String -> Int -> m (Either String IndexResult) + lookupByColumn _ _ _ = pure (Right NoIndex) -- \| 建一张空表 createTable :: String -> [(String, Column)] -> m (Either String ()) @@ -26,5 +68,24 @@ class (Monad m) => MonadStorage m where -- \| 删除表 dropTable :: String -> m (Either String ()) - -- \| 取全库快照 + -- \| 给某一列建索引(要求这一列整数取值唯一) + createIndex :: String -> String -> m (Either String ()) + createIndex _ _ = pure (Right ()) + + -- \| 去掉某一列的索引 + dropIndex :: String -> String -> m (Either String ()) + dropIndex _ _ = pure (Right ()) + + -- \| 取全库快照(表名 + 列 + 全部行) snapshot :: m Database + + -- \| 取全库**结构**:只要表名和列,不要行。 + -- + -- 语义检查和查询优化都只看列名/列类型,一行数据都不碰; + -- 而"取全库快照"要把所有行拉一遍,在单语句路径上是白花的钱。 + -- 默认实现退化成整库快照再抹掉行,有更便宜做法的实现(比如走数据字典)可以覆盖。 + schema :: m Database + schema = fmap (map withoutRows) snapshot + where + -- \| 抹掉行,只留表名与列定义 + withoutRows (name, t) = (name, t {tableRows = []}) diff --git a/chusql-engine/src/ChuSQL/Storage/IPC.hs b/chusql-engine/src/ChuSQL/Storage/IPC.hs index 3f31e00..b5574ba 100644 --- a/chusql-engine/src/ChuSQL/Storage/IPC.hs +++ b/chusql-engine/src/ChuSQL/Storage/IPC.hs @@ -6,11 +6,14 @@ module ChuSQL.Storage.IPC ( Request (..), Response (..), SchemaColumn (..), + TableInfo (..), sendRequest, closeConnection, doListTables, - doLookupByKey, - doInsertWith, + doLookupByColumn, + doInsert, + doInsertMany, + doDeleteKeys, doDescribeTable, doDropTable, doListCatalog, @@ -20,7 +23,7 @@ import ChuSQL.Model import ChuSQL.Storage import Control.Concurrent.MVar (MVar, modifyMVar, modifyMVar_, newMVar) import Control.Exception (IOException, try) -import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecodeStrict, encode, object, withObject, (.:), (.=)) +import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecodeStrict, encode, object, withObject, (.:), (.:?), (.!=), (.=)) import Data.Aeson qualified as A import Data.Aeson.Key qualified as K import Data.Aeson.KeyMap qualified as KM @@ -155,28 +158,41 @@ instance FromJSON SchemaColumn where data Request = ReqPing | ReqScan String - | ReqInsert String Row (Maybe Int) + | ReqInsert String Row + | ReqInsertBatch String [Row] + | ReqDeleteKeys String [Int] | ReqReplaceAll String [Row] | ReqListTables - | ReqLookupByKey String Int + | ReqLookupByColumn String String Int | ReqDescribeTable String | ReqCreateTable String [SchemaColumn] | ReqDropTable String + | ReqCreateIndex String String + | ReqDropIndex String String | ReqListCatalog -- | 请求编码 instance ToJSON Request where toJSON ReqPing = object ["method" .= ("ping" :: T.Text)] toJSON (ReqScan t) = object ["method" .= ("scan" :: T.Text), "table" .= t] - toJSON (ReqInsert t r mk) = - object $ + toJSON (ReqInsert t r) = + object [ "method" .= ("insert" :: T.Text) , "table" .= t , "row" .= rowToJSON r ] - ++ case mk of - Nothing -> [] - Just k -> ["key" .= k] + toJSON (ReqInsertBatch t rs) = + object + [ "method" .= ("insert_batch" :: T.Text) + , "table" .= t + , "rows" .= map rowToJSON rs + ] + toJSON (ReqDeleteKeys t ks) = + object + [ "method" .= ("delete_keys" :: T.Text) + , "table" .= t + , "keys" .= ks + ] toJSON (ReqReplaceAll t rs) = object [ "method" .= ("replace_all" :: T.Text) @@ -184,10 +200,11 @@ instance ToJSON Request where , "rows" .= map rowToJSON rs ] toJSON ReqListTables = object ["method" .= ("list_tables" :: T.Text)] - toJSON (ReqLookupByKey t k) = + toJSON (ReqLookupByColumn t c k) = object [ "method" .= ("lookup_by_index" :: T.Text) , "table" .= t + , "column" .= c , "key" .= k ] toJSON (ReqDescribeTable t) = @@ -203,17 +220,40 @@ instance ToJSON Request where ] toJSON (ReqDropTable t) = object ["method" .= ("drop_table" :: T.Text), "table" .= t] + toJSON (ReqCreateIndex t c) = + object + [ "method" .= ("create_index" :: T.Text) + , "table" .= t + , "column" .= c + ] + toJSON (ReqDropIndex t c) = + object + [ "method" .= ("drop_index" :: T.Text) + , "table" .= t + , "column" .= c + ] toJSON ReqListCatalog = object ["method" .= ("list_catalog" :: T.Text)] +-- | 一张表的线上信息:列 + 行数 + 索引 + 列统计 +data TableInfo = TableInfo + { tiTable :: String + , tiColumns :: [SchemaColumn] + , tiRows :: Int + , tiIndexes :: [String] + , tiStats :: [(String, Int, Bool)] + } + deriving (Show, Eq) + -- | 响应类型 data Response = RespPong | RespRows [Row] | RespTables [String] - | RespSchema [SchemaColumn] Int + | RespSchema TableInfo + | RespNoIndex | RespOk | RespError String - | RespCatalog [(String, [SchemaColumn], Int)] + | RespCatalog [TableInfo] -- | 响应解码 instance FromJSON Response where @@ -222,25 +262,35 @@ instance FromJSON Response where case status :: T.Text of "pong" -> pure RespPong "ok" -> pure RespOk + "no_index" -> pure RespNoIndex "rows" -> RespRows <$> (o .: "rows" >>= mapM rowFromJSON) "tables" -> RespTables <$> o .: "tables" "error" -> RespError <$> o .: "message" - "schema" -> do - cols <- o .: "columns" - cnt <- o .: "row_count" - pure (RespSchema cols cnt) + "schema" -> RespSchema <$> parseTable (A.Object o) "catalog" -> do - xs <- o .: "schemas" - parsed <- mapM parseTableSchema xs - pure (RespCatalog parsed) + xs <- o .: "schemas" :: Parser [A.Value] + RespCatalog <$> mapM parseTable xs other -> fail ("unknown status: " ++ T.unpack other) where - -- \| 解析一条 list_catalog 记录。 - parseTableSchema = withObject "TableSchema" $ \o -> do + -- \| 解一张表的 schema(新字段都给了默认值,老响应也能解) + parseTable :: A.Value -> Parser TableInfo + parseTable = withObject "TableInfo" $ \o -> do t <- o .: "table" cols <- o .: "columns" cnt <- o .: "row_count" - pure (t, cols, cnt) + idx <- o .:? "indexes" .!= [] + sts <- o .:? "stats" .!= [] + idxCols <- mapM (\v -> withObject "IndexWire" (.: "column") v) (idx :: [A.Value]) + stats <- mapM parseStat (sts :: [A.Value]) + pure (TableInfo t cols cnt idxCols stats) + + -- \| 解一条列统计 + parseStat :: A.Value -> Parser (String, Int, Bool) + parseStat = withObject "ColumnStat" $ \o -> do + name <- o .: "name" + distinct <- o .: "distinct" + capped <- o .:? "capped" .!= False + pure (name, distinct, capped) -- | 值编码成 JSON valueToJSON :: Value -> A.Value @@ -275,87 +325,88 @@ rowFromJSON _ = fail "row must be a JSON object" -- * 内部封装 +-- | 发一条请求,把响应翻成 Either。 +-- +-- 每个操作的区别只有两处:发什么请求、认哪种响应。所以固定套路收在这里: +-- 对面报错 → `Left`;认得这种响应 → `Right`;其余一律算协议错(比如对面版本对不上)。 +ask :: String -> Request -> (Response -> Maybe a) -> IO (Either String a) +ask what req recognize = do + resp <- sendRequest req + pure $ case resp of + RespError e -> Left e + other -> case recognize other of + Just a -> Right a + Nothing -> Left ("unexpected response to " ++ what) + +-- | 只认"成了"(多数写操作都这样) +okOnly :: Response -> Maybe () +okOnly RespOk = Just () +okOnly _ = Nothing + -- | 发 Scan doScan :: String -> IO (Either String [Row]) -doScan t = do - resp <- sendRequest (ReqScan t) - pure $ case resp of - RespRows rows -> Right rows - RespError e -> Left e - _ -> Left "unexpected response to scan" +doScan t = ask "scan" (ReqScan t) $ \resp -> case resp of + RespRows rows -> Just rows + _ -> Nothing --- | 发 Insert,可带索引键。 -doInsertWith :: String -> Row -> Maybe Int -> IO (Either String ()) -doInsertWith t r mk = do - resp <- sendRequest (ReqInsert t r mk) - pure $ case resp of - RespOk -> Right () - RespError e -> Left e - _ -> Left "unexpected response to insert" +-- | 发 Insert(索引由存储层自己维护,这里不传 key) +doInsert :: String -> Row -> IO (Either String ()) +doInsert t r = ask "insert" (ReqInsert t r) okOnly + +-- | 发 InsertBatch:一批行一次请求 +doInsertMany :: String -> [Row] -> IO (Either String ()) +doInsertMany t rs = ask "insert_batch" (ReqInsertBatch t rs) okOnly + +-- | 发 DeleteKeys +doDeleteKeys :: String -> [Int] -> IO (Either String ()) +doDeleteKeys t ks = ask "delete_keys" (ReqDeleteKeys t ks) okOnly -- | 发 ReplaceAll doReplaceAll :: String -> [Row] -> IO (Either String ()) -doReplaceAll t rs = do - resp <- sendRequest (ReqReplaceAll t rs) - pure $ case resp of - RespOk -> Right () - RespError e -> Left e - _ -> Left "unexpected response to replace_all" +doReplaceAll t rs = ask "replace_all" (ReqReplaceAll t rs) okOnly -- | 发 ListTables doListTables :: IO (Either String [String]) -doListTables = do - resp <- sendRequest ReqListTables - pure $ case resp of - RespTables ts -> Right ts - RespError e -> Left e - _ -> Left "unexpected response to list_tables" +doListTables = ask "list_tables" ReqListTables $ \resp -> case resp of + RespTables ts -> Just ts + _ -> Nothing --- | 一次拿全库 schema(表名 + 列 + 行数)。 -doListCatalog :: IO (Either String [(String, [SchemaColumn], Int)]) -doListCatalog = do - resp <- sendRequest ReqListCatalog - pure $ case resp of - RespCatalog xs -> Right xs - RespError e -> Left e - _ -> Left "unexpected response to list_catalog" +-- | 一次拿全库 schema(表名 + 列 + 行数 + 索引 + 统计)。 +doListCatalog :: IO (Either String [TableInfo]) +doListCatalog = ask "list_catalog" ReqListCatalog $ \resp -> case resp of + RespCatalog xs -> Just xs + _ -> Nothing --- | 发 LookupByIndex -doLookupByKey :: String -> Int -> IO (Either String (Maybe Row)) -doLookupByKey t k = do - resp <- sendRequest (ReqLookupByKey t k) - pure $ case resp of - RespRows [] -> Right Nothing - RespRows (r : _) -> Right (Just r) - RespError e -> Left e - _ -> Left "unexpected response to lookup_by_index" +-- | 按某一列的索引取一行。 +-- 返回 `NoIndex` 表示"这个列上没有索引",上层据此退回全表扫描。 +doLookupByColumn :: String -> String -> Int -> IO (Either String IndexResult) +doLookupByColumn t c k = ask "lookup_by_index" (ReqLookupByColumn t c k) $ \resp -> case resp of + RespRows [] -> Just (IndexRow Nothing) + RespRows (r : _) -> Just (IndexRow (Just r)) + RespNoIndex -> Just NoIndex + _ -> Nothing -- | 发 DescribeTable -doDescribeTable :: String -> IO (Either String ([SchemaColumn], Int)) -doDescribeTable t = do - resp <- sendRequest (ReqDescribeTable t) - pure $ case resp of - RespSchema cols n -> Right (cols, n) - RespError e -> Left e - _ -> Left "unexpected response to describe_table" +doDescribeTable :: String -> IO (Either String TableInfo) +doDescribeTable t = ask "describe_table" (ReqDescribeTable t) $ \resp -> case resp of + RespSchema info -> Just info + _ -> Nothing + +-- | 给某一列建索引 +doCreateIndex :: String -> String -> IO (Either String ()) +doCreateIndex t c = ask "create_index" (ReqCreateIndex t c) okOnly + +-- | 去掉某一列的索引 +doDropIndex :: String -> String -> IO (Either String ()) +doDropIndex t c = ask "drop_index" (ReqDropIndex t c) okOnly -- | 发 CreateTable doCreateTable :: String -> [SchemaColumn] -> IO (Either String ()) -doCreateTable t cols = do - resp <- sendRequest (ReqCreateTable t cols) - pure $ case resp of - RespOk -> Right () - RespError e -> Left e - _ -> Left "unexpected response to create_table" +doCreateTable t cols = ask "create_table" (ReqCreateTable t cols) okOnly -- | 发 DropTable doDropTable :: String -> IO (Either String ()) -doDropTable t = do - resp <- sendRequest (ReqDropTable t) - pure $ case resp of - RespOk -> Right () - RespError e -> Left e - _ -> Left "unexpected response to drop_table" +doDropTable t = ask "drop_table" (ReqDropTable t) okOnly -- * 表结构 @@ -375,14 +426,20 @@ instance MonadStorage IPCStorage where -- \| 发 Scan scan t = IPCStorage (doScan t) - -- \| 发 Insert - insert t r mk = IPCStorage (doInsertWith t r mk) + -- \| 发 Insert(索引由存储层自己维护) + insert t r = IPCStorage (doInsert t r) + + -- \| 一批行一次请求:N 行只落一次盘 + insertMany t rs = IPCStorage (doInsertMany t rs) + + -- \| 按 id 批量删行 + deleteKeys t ks = IPCStorage (doDeleteKeys t ks) -- \| 发 ReplaceAll replaceAll t rs = IPCStorage (doReplaceAll t rs) - -- \| 发 LookupByIndex - lookupByKey t k = IPCStorage (doLookupByKey t k) + -- \| 按某一列的索引取一行(没有索引就回 NoIndex) + lookupByColumn t c k = IPCStorage (doLookupByColumn t c k) -- \| 发 CreateTable createTable name cols = IPCStorage (doCreateTable name (map toWire cols)) @@ -395,6 +452,12 @@ instance MonadStorage IPCStorage where -- \| 发 DropTable dropTable name = IPCStorage (doDropTable name) + -- \| 建索引 + createIndex t c = IPCStorage (doCreateIndex t c) + + -- \| 删索引 + dropIndex t c = IPCStorage (doDropIndex t c) + -- \| 列表 + 逐表扫描拼库 snapshot = IPCStorage $ do result <- doListCatalog @@ -403,6 +466,19 @@ instance MonadStorage IPCStorage where Right xs -> mapM loadEntry xs where -- \| 加载一张表(schema 来自 catalog,行来自 scan) - loadEntry (t, cols, _n) = do - rows <- doScan t - pure (t, Table t (schemaToColumns cols) (either (const []) id rows)) + loadEntry info = do + rows <- doScan (tiTable info) + pure + ( tiTable info + , Table (tiTable info) (schemaToColumns (tiColumns info)) (either (const []) id rows) + ) + + -- \| 只问数据字典要结构,一行数据都不拉过来 + schema = IPCStorage $ do + result <- doListCatalog + pure $ case result of + Left _ -> [] + Right xs -> + [ (tiTable i, Table (tiTable i) (schemaToColumns (tiColumns i)) []) + | i <- xs + ] diff --git a/chusql-engine/src/ChuSQL/Storage/Memory.hs b/chusql-engine/src/ChuSQL/Storage/Memory.hs index 41f1bf1..ca838ee 100644 --- a/chusql-engine/src/ChuSQL/Storage/Memory.hs +++ b/chusql-engine/src/ChuSQL/Storage/Memory.hs @@ -2,9 +2,11 @@ module ChuSQL.Storage.Memory (MemoryStorage (runMemoryStorage)) where import ChuSQL.Model import ChuSQL.Storage -import Data.List (find) -- 内存实现:状态是 Database,错误通道是 Either String。 +-- +-- 它只实现"必须有"的那几个:批量插入、行级删除、索引、统计全走类的默认实现 +-- (内存里数据本来就在手上,扫一遍就是最快的做法)。 -- * 实例 @@ -39,7 +41,7 @@ instance MonadStorage MemoryStorage where Right (Right (tableRows tbl), db) -- \| 追加一行 - insert t r _ = MemoryStorage $ \db -> do + insert t r = MemoryStorage $ \db -> do tbl <- lookupTable db t Right (Right (), replaceTable t tbl{tableRows = tableRows tbl ++ [r]} db) @@ -48,11 +50,6 @@ instance MonadStorage MemoryStorage where tbl <- lookupTable db t Right (Right (), replaceTable t tbl{tableRows = rows} db) - -- \| 按主键找第一行 - lookupByKey t k = MemoryStorage $ \db -> do - tbl <- lookupTable db t - Right (Right (findRow tbl k), db) - -- \| 建空表;已存在报错 createTable name cols = MemoryStorage $ \db -> if any ((== name) . fst) db @@ -70,14 +67,8 @@ instance MonadStorage MemoryStorage where -- * 工具 --- | 按 id 列找一行 -findRow :: Table -> Int -> Maybe Row -findRow tbl k = find matches (tableRows tbl) - where - -- \| 这行的 id 是否等于 k - matches r = lookup "id" r == Just (VInt k) - -- | 用给定表替换同名表 replaceTable :: String -> Table -> Database -> Database replaceTable name table = map (\(n, t) -> if n == name then (n, table) else (n, t)) + diff --git a/chusql-engine/src/ChuSQL/Syntax/AST.hs b/chusql-engine/src/ChuSQL/Syntax/AST.hs index 9398e35..a5c02c0 100644 --- a/chusql-engine/src/ChuSQL/Syntax/AST.hs +++ b/chusql-engine/src/ChuSQL/Syntax/AST.hs @@ -6,7 +6,7 @@ import ChuSQL.Model (Column (..)) -- * 语句 --- | 一条语句:查 / 加 / 删 / 改 / 建表 / 删表 +-- | 一条语句:查 / 加 / 删 / 改 / 建表 / 删表 / 建索引 / 删索引 data Statement = Select { selectCols :: [String] @@ -15,11 +15,15 @@ data Statement , selectOrderBy :: [(String, SortDir)] , selectLimit :: Maybe Int } - | Insert String [String] [Expr] + | -- \| 表、列、若干行(`VALUES (...), (...)` 可以一次给多行) + Insert String [String] [[Expr]] | Delete String (Maybe Expr) | Update String [(String, Expr)] (Maybe Expr) | CreateTable String [(String, Column)] | DropTable String + | -- \| 给某一列建索引(列名即索引名) + CreateIndex String String + | DropIndex String String deriving (Show, Eq) -- | 数据来源:单表,或两表拼接 diff --git a/chusql-engine/src/ChuSQL/Syntax/Parser.hs b/chusql-engine/src/ChuSQL/Syntax/Parser.hs index 4cbd4f5..34102f0 100644 --- a/chusql-engine/src/ChuSQL/Syntax/Parser.hs +++ b/chusql-engine/src/ChuSQL/Syntax/Parser.hs @@ -171,6 +171,8 @@ reservedWords = , "set" , "create" , "table" + , "index" + , "drop" ] {- | 表别名。别名不能是保留字:否则 @FROM users WHERE age > 18@ 会把 @WHERE@ @@ -232,6 +234,26 @@ dropTableStatement = do name <- identifier return (DropTable name) +-- | 读 CREATE INDEX(列名即索引名:一列最多一个索引) +createIndexStatement :: Parser Statement +createIndexStatement = do + keyword "create" + keyword "index" + keyword "on" + tbl <- identifier + col <- between (symbol "(") (symbol ")") identifier + return (CreateIndex tbl col) + +-- | 读 DROP INDEX +dropIndexStatement :: Parser Statement +dropIndexStatement = do + keyword "drop" + keyword "index" + keyword "on" + tbl <- identifier + col <- between (symbol "(") (symbol ")") identifier + return (DropIndex tbl col) + -- | 建表时的一列 columnDef :: Parser (String, Column) columnDef = do @@ -270,7 +292,7 @@ selectStatement = do , selectLimit = mLimit } --- | 读 INSERT +-- | 读 INSERT:`VALUES (...)` 后面可以跟多个括号,一次插多行 insertStatement :: Parser Statement insertStatement = do keyword "INSERT" @@ -282,12 +304,12 @@ insertStatement = do (symbol ")") (sepBy1 identifier (symbol ",")) keyword "VALUES" - vals <- - between - (symbol "(") - (symbol ")") - (sepBy1 atom (symbol ",")) - return (Insert tbl cols vals) + rows <- sepBy1 valueRow (symbol ",") + return (Insert tbl cols rows) + +-- | 一行值:一对括号里的若干字面量 +valueRow :: Parser [Expr] +valueRow = between (symbol "(") (symbol ")") (sepBy1 atom (symbol ",")) -- | 读 DELETE deleteStatement :: Parser Statement @@ -316,11 +338,16 @@ parseStatement input = Left err -> Left (errorBundlePretty err) Right q -> Right q where - -- \| 依次尝试各种语句 + -- \| 依次尝试各种语句。 + -- `CREATE` / `DROP` 后面跟的词决定是哪条语句,而 `keyword` 的 `try` 只能回溯到 + -- "这个词之前"——`CREATE` 一旦被吃掉就没得退了。所以带 `INDEX` 的那两个各自 + -- 用 `try` 包住并排在前面:`CREATE INDEX ...` 先试,退回来再试 `CREATE TABLE ...`。 statementP = selectStatement <|> insertStatement <|> deleteStatement <|> updateStatement - <|> createTableStatement + <|> try createIndexStatement + <|> try createTableStatement + <|> try dropIndexStatement <|> dropTableStatement diff --git a/chusql-engine/test/Spec.hs b/chusql-engine/test/Spec.hs index b899f85..efe726f 100644 --- a/chusql-engine/test/Spec.hs +++ b/chusql-engine/test/Spec.hs @@ -6,7 +6,7 @@ import ChuSQL.Algebra.Optimize (optimize, pushProject) import ChuSQL.Algebra.Planner (translate) import ChuSQL.Engine import ChuSQL.Model -import ChuSQL.Storage (MonadStorage (..)) +import ChuSQL.Storage (IndexResult (..), MonadStorage (..)) import ChuSQL.Storage.IPC (IPCStorage (runIPCStorage), Request (ReqPing), Response (RespPong), doListTables, sendRequest) import ChuSQL.Syntax.AST import ChuSQL.Syntax.Parser @@ -299,18 +299,18 @@ main = hspec $ do describe "ChuSQL.Engine (INSERT)" $ do it "parses a simple INSERT" $ do parseStatement "INSERT INTO users (name, age) VALUES ('Dave', 22)" - `shouldBe` Right (Insert "users" ["name", "age"] [LitStr "Dave", LitInt 22]) + `shouldBe` Right (Insert "users" ["name", "age"] [[LitStr "Dave", LitInt 22]]) it "parses INSERT with a single column" $ do parseStatement "INSERT INTO users (name) VALUES ('Eve')" - `shouldBe` Right (Insert "users" ["name"] [LitStr "Eve"]) + `shouldBe` Right (Insert "users" ["name"] [[LitStr "Eve"]]) it "parses INSERT case-insensitively" $ do parseStatement "insert into users (name, age) values ('Dave', 22)" - `shouldBe` Right (Insert "users" ["name", "age"] [LitStr "Dave", LitInt 22]) + `shouldBe` Right (Insert "users" ["name", "age"] [[LitStr "Dave", LitInt 22]]) it "executes INSERT and adds a row at the end" $ do - case runStatement testDB (Insert "users" ["name", "age"] [LitStr "Dave", LitInt 22]) of + case runStatement testDB (Insert "users" ["name", "age"] [[LitStr "Dave", LitInt 22]]) of Left err -> expectationFailure err Right (db', _) -> do rowsOf (runStatement db' (makeSelect ["name", "age"] "users" Nothing)) @@ -322,7 +322,7 @@ main = hspec $ do ] it "does not modify the original database" $ do - case runStatement testDB (Insert "users" ["name", "age"] [LitStr "Dave", LitInt 22]) of + case runStatement testDB (Insert "users" ["name", "age"] [[LitStr "Dave", LitInt 22]]) of Left err -> expectationFailure err Right (_, _) -> do rowsOf (runStatement testDB (makeSelect ["name"] "users" Nothing)) @@ -331,15 +331,15 @@ main = hspec $ do Left _ -> False it "returns Left when column count does not match value count" $ do - runStatement testDB (Insert "users" ["name", "age"] [LitStr "Dave"]) + runStatement testDB (Insert "users" ["name", "age"] [[LitStr "Dave"]]) `shouldSatisfy` isLeft it "returns Left for an unknown table" $ do - runStatement testDB (Insert "nonexistent" ["name"] [LitStr "X"]) + runStatement testDB (Insert "nonexistent" ["name"] [[LitStr "X"]]) `shouldSatisfy` isLeft it "returns Left when a value has the wrong type for comparison" $ do - runStatement testDB (Insert "users" ["name"] [Col "other"]) + runStatement testDB (Insert "users" ["name"] [[Col "other"]]) `shouldSatisfy` isLeft it "writes id column to index and queries it back" $ do case parseStatement "INSERT INTO users (id, name, age) VALUES (42, 'Zoe', 20)" >>= runStatement testDB of @@ -347,6 +347,49 @@ main = hspec $ do Right (db', _) -> do rowsOf (parseStatement "SELECT name FROM users WHERE id = 42" >>= runStatement db') `shouldBe` Right [[("name", VStr "Zoe")]] + + it "parses multi-row INSERT into one statement" $ do + parseStatement "INSERT INTO users (id, name) VALUES (10, 'A'), (11, 'B'), (12, 'C')" + `shouldBe` Right + ( Insert + "users" + ["id", "name"] + [ [LitInt 10, LitStr "A"] + , [LitInt 11, LitStr "B"] + , [LitInt 12, LitStr "C"] + ] + ) + + it "inserts all rows of a multi-row INSERT" $ do + case parseStatement "INSERT INTO users (id, name, age) VALUES (7, 'A', 1), (8, 'B', 2)" >>= runStatement testDB of + Left err -> expectationFailure err + Right (db', _) -> do + rowsOf (parseStatement "SELECT name FROM users WHERE age < 3" >>= runStatement db') + `shouldBe` Right [[("name", VStr "A")], [("name", VStr "B")]] + -- 多行里只要有一行列数不对,整条都要拒绝 + runStatement testDB (Insert "users" ["id", "name"] [[LitInt 1, LitStr "A"], [LitInt 2]]) + `shouldSatisfy` isLeft + + describe "ChuSQL.Engine (INDEX)" $ do + it "parses CREATE INDEX and DROP INDEX" $ do + parseStatement "CREATE INDEX ON users (age)" + `shouldBe` Right (CreateIndex "users" "age") + parseStatement "DROP INDEX ON users (age)" + `shouldBe` Right (DropIndex "users" "age") + + it "rejects an index on an unknown column" $ do + (parseStatement "CREATE INDEX ON users (nope)" >>= runStatement testDB) + `shouldSatisfy` isLeft + + it "rejects an index on an unknown table" $ do + (parseStatement "CREATE INDEX ON nope (age)" >>= runStatement testDB) + `shouldSatisfy` isLeft + + it "accepts index DDL in the memory storage (no-op there)" $ do + rowsOf (parseStatement "CREATE INDEX ON users (age)" >>= runStatement testDB) + `shouldBe` Right [] + rowsOf (parseStatement "DROP INDEX ON users (age)" >>= runStatement testDB) + `shouldBe` Right [] describe "ChuSQL.Engine (DELETE)" $ do it "parses DELETE with WHERE" $ do parseStatement "DELETE FROM users WHERE age < 18" @@ -961,10 +1004,27 @@ main = hspec $ do sameResultAsUnoptimized "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id ORDER BY o.product DESC LIMIT 2" it "rewrites Filter id = k into Lookup" $ do fmap planRoot (optimizedPlan "SELECT name FROM users WHERE id = 2") - `shouldBe` Right (Lookup "users" 2) + `shouldBe` Right (Lookup Nothing "users" "id" 2) it "keeps Lookup result identical to unoptimized" $ do sameResultAsUnoptimized "SELECT name FROM users WHERE id = 2" + it "rewrites an equality on any column into Lookup" $ do + -- 能不能真走索引由存储层回答;优化器只负责说"这里可以点查" + fmap planRoot (optimizedPlan "SELECT name FROM users WHERE age = 25") + `shouldBe` Right (Lookup Nothing "users" "age" 25) + + it "keeps a point lookup on a non-indexed column identical to unoptimized" $ do + -- 内存实现没有索引,这条会走"退回扫描"的分支 + sameResultAsUnoptimized "SELECT name FROM users WHERE age = 25" + + it "keeps an aliased point lookup identical to unoptimized" $ do + -- 别名要留住:回来的一行必须和 Scan 别名 表 长得一样,不然投影取不到列 + sameResultAsUnoptimized "SELECT u.name FROM users u WHERE u.id = 1" + + it "returns the row for an aliased point lookup" $ do + rowsOf (parseStatement "SELECT u.name FROM users u WHERE u.id = 1" >>= runStatement testDB) + `shouldBe` Right [[("u.name", VStr "Alice")]] + describe "ChuSQL.Semantic" $ do it "rejects an unknown column in WHERE" $ do (parseStatement "SELECT name FROM users WHERE nope > 18" >>= runStatement testDB) @@ -1108,7 +1168,7 @@ main = hspec $ do it "scan returns rows with all three value types after insert" $ do withTestServer $ \srv -> withServerEnv srv $ do let row = [("id", VInt 7), ("name", VStr "Alice"), ("flag", VBool True)] - runIPCStorage (insert "users" row (Just 7)) `shouldReturn` Right () + runIPCStorage (insert "users" row) `shouldReturn` Right () rows <- runIPCStorage (scan "users") (map sortRow <$> rows) `shouldBe` Right [sortRow row] @@ -1119,7 +1179,7 @@ main = hspec $ do it "scan returns all 20 inserted rows in order" $ do withTestServer $ \srv -> withServerEnv srv $ do - mapM_ (\i -> runIPCStorage (insert "many" [("id", VInt i)] (Just i))) [1 .. 20 :: Int] + mapM_ (\i -> runIPCStorage (insert "many" [("id", VInt i)])) [1 .. 20 :: Int] rows <- runIPCStorage (scan "many") fmap length rows `shouldBe` Right 20 fmap (sortOn show . map (lookup "id")) rows @@ -1127,35 +1187,35 @@ main = hspec $ do it "replaceAll leaves only the new rows" $ do withTestServer $ \srv -> withServerEnv srv $ do - runIPCStorage (insert "t" [("id", VInt 1)] (Just 1)) `shouldReturn` Right () + runIPCStorage (insert "t" [("id", VInt 1)]) `shouldReturn` Right () runIPCStorage (replaceAll "t" [[("id", VInt 9)], [("id", VInt 8)]]) `shouldReturn` Right () rows <- runIPCStorage (scan "t") fmap (sortOn show . map (lookup "id")) rows `shouldBe` Right (sortOn show [Just (VInt 9), Just (VInt 8)]) - it "lookupByKey finds a row inserted with an id over the pipe" $ do + it "lookupByColumn finds a row inserted with an id over the pipe" $ do withTestServer $ \srv -> withServerEnv srv $ do let row = [("id", VInt 7), ("name", VStr "Zoe")] - runIPCStorage (insert "keyed" row (Just 7)) `shouldReturn` Right () - found <- runIPCStorage (lookupByKey "keyed" 7) - fmap (fmap sortRow) found `shouldBe` Right (Just (sortRow row)) - missing <- runIPCStorage (lookupByKey "keyed" 8) - missing `shouldBe` Right Nothing + runIPCStorage (insert "keyed" row) `shouldReturn` Right () + found <- runIPCStorage (lookupByColumn "keyed" "id" 7) + fmap sortRow (indexRow found) `shouldBe` Just (sortRow row) + missing <- runIPCStorage (lookupByColumn "keyed" "id" 8) + missing `shouldBe` Right (IndexRow Nothing) it "replaceAll rebuilds the index over the pipe" $ do withTestServer $ \srv -> withServerEnv srv $ do - runIPCStorage (insert "rb" [("id", VInt 1)] (Just 1)) `shouldReturn` Right () + runIPCStorage (insert "rb" [("id", VInt 1)]) `shouldReturn` Right () runIPCStorage (replaceAll "rb" [[("id", VInt 9), ("name", VStr "Zoe")]]) `shouldReturn` Right () - old <- runIPCStorage (lookupByKey "rb" 1) - old `shouldBe` Right Nothing - new <- runIPCStorage (lookupByKey "rb" 9) - fmap (fmap sortRow) new - `shouldBe` Right (Just (sortRow [("id", VInt 9), ("name", VStr "Zoe")])) + old <- runIPCStorage (lookupByColumn "rb" "id" 1) + old `shouldBe` Right (IndexRow Nothing) + new <- runIPCStorage (lookupByColumn "rb" "id" 9) + fmap sortRow (indexRow new) + `shouldBe` Just (sortRow [("id", VInt 9), ("name", VStr "Zoe")]) it "snapshot lists tables and scans each one to build the database" $ do withTestServer $ \srv -> withServerEnv srv $ do - runIPCStorage (insert "a" [("id", VInt 1)] (Just 1)) `shouldReturn` Right () - runIPCStorage (insert "b" [("id", VInt 2)] (Just 2)) `shouldReturn` Right () + runIPCStorage (insert "a" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (insert "b" [("id", VInt 2)]) `shouldReturn` Right () db <- runIPCStorage snapshot map fst db `shouldBe` ["a", "b"] map (length . tableRows . snd) db `shouldBe` [1, 1] @@ -1163,8 +1223,8 @@ main = hspec $ do it "doListTables returns the tables that exist" $ do withTestServer $ \srv -> withServerEnv srv $ do - runIPCStorage (insert "t1" [("id", VInt 1)] (Just 1)) `shouldReturn` Right () - runIPCStorage (insert "t2" [("id", VInt 2)] (Just 2)) `shouldReturn` Right () + runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right () doListTables `shouldReturn` Right ["t1", "t2"] it "rows survive a restart on the same data directory" $ do @@ -1174,7 +1234,7 @@ main = hspec $ do Right bin -> do srv1 <- startServer bin let dir = dataDir srv1 - withServerEnv srv1 $ runIPCStorage (insert "persist" [("id", VInt 42)] (Just 42)) `shouldReturn` Right () + withServerEnv srv1 $ runIPCStorage (insert "persist" [("id", VInt 42)]) `shouldReturn` Right () stopServer srv1 srv2 <- startServerAt bin dir rows <- withServerEnv srv2 $ runIPCStorage (scan "persist") @@ -1196,7 +1256,7 @@ main = hspec $ do _ <- runIPCStorage ( runStatementM - (Insert "ipc_idx" ["id", "name"] [LitInt 42, LitStr "Zoe"]) + (Insert "ipc_idx" ["id", "name"] [[LitInt 42, LitStr "Zoe"]]) ) result <- runIPCStorage @@ -1211,6 +1271,76 @@ main = hspec $ do ) ) result `shouldBe` Right [[("name", VStr "Zoe")]] + + it "CREATE INDEX makes a query on that column use the index" $ do + withTestServer $ \srv -> withServerEnv srv $ do + _ <- runIPCStorage (runStatementM (CreateTable "ix_t" [("id", TInt), ("code", TInt)])) + _ <- + runIPCStorage + ( runStatementM + (Insert "ix_t" ["id", "code"] [[LitInt 1, LitInt 500], [LitInt 2, LitInt 501]]) + ) + + -- 还没建索引:存储层明确回"我帮不上忙",上层会退回扫描 + withoutIndex <- runIPCStorage (lookupByColumn "ix_t" "code" 501) + withoutIndex `shouldBe` Right NoIndex + scanned <- runIPCStorage (runStatementM (makeSelect ["code"] "ix_t" (Just (Eq (Col "code") (LitInt 501))))) + scanned `shouldBe` Right [[("code", VInt 501)]] + + -- 建索引之后就真的走索引了 + created <- runIPCStorage (runStatementM (CreateIndex "ix_t" "code")) + created `shouldBe` Right [] + withIndex <- runIPCStorage (lookupByColumn "ix_t" "code" 501) + fmap sortRow (indexRow withIndex) + `shouldBe` Just (sortRow [("id", VInt 2), ("code", VInt 501)]) + indexed <- runIPCStorage (runStatementM (makeSelect ["code"] "ix_t" (Just (Eq (Col "code") (LitInt 501))))) + indexed `shouldBe` Right [[("code", VInt 501)]] + + -- 建完索引之后插入的行也要进索引 + _ <- runIPCStorage (runStatementM (Insert "ix_t" ["id", "code"] [[LitInt 3, LitInt 502]])) + fresh <- runIPCStorage (lookupByColumn "ix_t" "code" 502) + fmap sortRow (indexRow fresh) + `shouldBe` Just (sortRow [("id", VInt 3), ("code", VInt 502)]) + + it "CREATE INDEX over the pipe rejects a column with duplicates" $ do + withTestServer $ \srv -> withServerEnv srv $ do + _ <- runIPCStorage (runStatementM (CreateTable "dup_ix" [("id", TInt), ("age", TInt)])) + _ <- + runIPCStorage + ( runStatementM + (Insert "dup_ix" ["id", "age"] [[LitInt 1, LitInt 30], [LitInt 2, LitInt 30]]) + ) + res <- runIPCStorage (runStatementM (CreateIndex "dup_ix" "age")) + res `shouldSatisfy` isLeft + + it "DELETE removes only the matching rows over the pipe" $ do + withTestServer $ \srv -> withServerEnv srv $ do + _ <- runIPCStorage (runStatementM (CreateTable "del_t" [("id", TInt), ("name", TStr)])) + _ <- + runIPCStorage + ( runStatementM + ( Insert + "del_t" + ["id", "name"] + [[LitInt 1, LitStr "A"], [LitInt 2, LitStr "B"], [LitInt 3, LitStr "C"]] + ) + ) + _ <- runIPCStorage (runStatementM (Delete "del_t" (Just (Eq (Col "id") (LitInt 2))))) + rows <- runIPCStorage (runStatementM (makeSelect ["name"] "del_t" Nothing)) + rows `shouldBe` Right [[("name", VStr "A")], [("name", VStr "C")]] + gone <- runIPCStorage (lookupByColumn "del_t" "id" 2) + gone `shouldBe` Right (IndexRow Nothing) + + it "multi-row INSERT goes over the pipe in one statement" $ do + withTestServer $ \srv -> withServerEnv srv $ do + _ <- runIPCStorage (runStatementM (CreateTable "many_t" [("id", TInt)])) + _ <- + runIPCStorage + ( runStatementM + (Insert "many_t" ["id"] [[LitInt i] | i <- [1 .. 50]]) + ) + rows <- runIPCStorage (runStatementM (makeSelect ["id"] "many_t" Nothing)) + fmap length rows `shouldBe` Right 50 it "parses CREATE TABLE with two columns" $ do parseStatement "CREATE TABLE users (id INT, name TEXT)" `shouldBe` Right (CreateTable "users" [("id", TInt), ("name", TStr)]) @@ -1392,6 +1522,11 @@ withPipeName name act = do sortRow :: Row -> Row sortRow = sortOn fst +-- | 从一次索引点查的结果里取出那一行(没索引、没查到都给 Nothing) +indexRow :: Either String IndexResult -> Maybe Row +indexRow (Right (IndexRow r)) = r +indexRow _ = Nothing + isLeft :: Either a b -> Bool isLeft (Left _) = True isLeft (Right _) = False @@ -1428,7 +1563,7 @@ firstFilterCond (Join l r _) = case firstFilterCond l of Just p -> Just p Nothing -> firstFilterCond r firstFilterCond (Scan _ _) = Nothing -firstFilterCond (Lookup _ _) = Nothing +firstFilterCond (Lookup _ _ _ _) = Nothing anyFilter :: RelOp -> Bool anyFilter (Filter _ _) = True @@ -1437,7 +1572,7 @@ anyFilter (Sort _ x) = anyFilter x anyFilter (Limit _ x) = anyFilter x anyFilter (Join l r _) = anyFilter l || anyFilter r anyFilter (Scan _ _) = False -anyFilter (Lookup _ _) = False +anyFilter (Lookup _ _ _ _) = False stripProjects :: RelOp -> RelOp stripProjects (Project _ x) = stripProjects x @@ -1446,7 +1581,7 @@ stripProjects (Sort spec x) = Sort spec (stripProjects x) stripProjects (Limit n x) = Limit n (stripProjects x) stripProjects (Join l r c) = Join (stripProjects l) (stripProjects r) c stripProjects (Scan a t) = Scan a t -stripProjects (Lookup t k) = Lookup t k +stripProjects (Lookup a t c k) = Lookup a t c k projectedPlan :: String -> Either String RelOp projectedPlan sql = do diff --git a/chusql-storage/src/bin/server.rs b/chusql-storage/src/bin/server.rs index 18c397b..ba7985b 100644 --- a/chusql-storage/src/bin/server.rs +++ b/chusql-storage/src/bin/server.rs @@ -9,26 +9,25 @@ use chusql_storage::catalog::Catalog; use chusql_storage::config::{self, Config, Loaded}; use chusql_storage::heap::HeapTable; use chusql_storage::log; -use chusql_storage::protocol::{Request, Response, SchemaColumn, TableSchemaWire}; +use chusql_storage::protocol::{ + ColumnStatWire, IndexWire, Request, Response, SchemaColumn, TableSchemaWire, +}; use chusql_storage::wal::{Wal, WalOp}; use chusql_storage::{log_debug, log_error, log_info, log_warn}; -use interprocess::local_socket::{ - prelude::*, - GenericNamespaced, - ListenerOptions, -}; +use interprocess::local_socket::{prelude::*, GenericNamespaced, ListenerOptions}; use interprocess::TryClone; // 命名管道服务:按行读写 JSON,Windows 上映射成 \\.\pipe\。 // 启动时读配置并把每项来源打进日志;表句柄按名字缓存复用。 // 服务器 -/// 服务器状态:配置 + 表句柄缓存 + 写锁 +/// 服务器状态:配置 + 表句柄缓存 + 写锁 + 数据字典 struct Server { cfg: Config, tables: Mutex>>>, write_lock: Mutex<()>, catalog: Mutex, + wal: Wal, } impl Server { @@ -36,23 +35,34 @@ impl Server { fn new(cfg: Config) -> std::io::Result { std::fs::create_dir_all(&cfg.data_dir)?; let catalog = Catalog::load(cfg.data_dir.join("catalog.json"))?; + let wal = Wal::new(cfg.data_dir.join("wal.log")); Ok(Server { cfg, tables: Mutex::new(HashMap::new()), write_lock: Mutex::new(()), catalog: Mutex::new(catalog), + wal, }) } -// 路径 + // 路径 /// 表名 -> 数据文件 fn table_path(&self, table: &str) -> PathBuf { self.cfg.data_dir.join(format!("{}.db", table)) } - /// 表名 -> 索引文件 - fn index_path(&self, table: &str) -> PathBuf { - self.cfg.data_dir.join(format!("{}.idx", table)) + /// (表名, 列名) -> 索引文件。 + /// + /// `id` 是内建索引,沿用老文件名 `表名.idx`(老数据目录不用迁移); + /// 其余列是 `表名.列名.idx`。 + fn index_path(&self, table: &str, column: &str) -> PathBuf { + if column == "id" { + self.cfg.data_dir.join(format!("{}.idx", table)) + } else { + self.cfg + .data_dir + .join(format!("{}.{}.idx", table, column)) + } } /// 数据字典文件 @@ -60,13 +70,7 @@ impl Server { self.cfg.data_dir.join("catalog.json") } - /// WAL 文件 - fn wal_path(&self) -> PathBuf { - self.cfg.data_dir.join("wal.log") - } - - -// 表句柄 + // 表句柄 /// 表文件存在且非空 fn table_exists(&self, table: &str) -> bool { if self.tables.lock().unwrap().contains_key(table) { @@ -82,16 +86,40 @@ impl Server { .unwrap_or(false) } + /// 一张表该开哪些索引:内建的 `id`,加上数据字典里记的那些 + fn index_columns(&self, table: &str) -> Vec { + let mut cols = vec!["id".to_string()]; + let c = self.catalog.lock().unwrap(); + if let Some(s) = c.describe(table) { + for i in &s.indexes { + if !cols.contains(&i.column) { + cols.push(i.column.clone()); + } + } + } + cols + } + /// 拿表句柄(缓存复用) fn table_handle(&self, table: &str) -> std::io::Result>> { + if let Some(h) = self.tables.lock().unwrap().get(table) { + return Ok(h.clone()); + } + // 先把索引清单取出来,再进 tables 锁(别套着锁去要 catalog) + let columns = self.index_columns(table); + let paths: Vec<(String, PathBuf)> = columns + .iter() + .map(|c| (c.clone(), self.index_path(table, c))) + .collect(); + let mut map = self.tables.lock().unwrap(); if let Some(h) = map.get(table) { return Ok(h.clone()); } std::fs::create_dir_all(&self.cfg.data_dir)?; - let t = HeapTable::open_indexed( + let t = HeapTable::open_with_indexes( self.table_path(table), - self.index_path(table), + &paths, self.cfg.page_size, self.cfg.btree_order, self.cfg.pool_size, @@ -112,15 +140,6 @@ impl Server { f(&mut guard) } - /// 同上,但一定带索引 - fn with_indexed_table( - &self, - table: &str, - f: impl FnOnce(&mut HeapTable) -> std::io::Result, - ) -> std::io::Result { - self.with_table(table, f) - } - /// 表不存在就报错 fn with_existing_table( &self, @@ -138,32 +157,45 @@ impl Server { self.with_table(table, f) } -// 写入路径 + // 写入路径 /// 把一条 WAL 操作落到磁盘 fn apply_op(&self, op: &WalOp) -> std::io::Result<()> { match op { - WalOp::Insert { table, row, key } => { - match key { - None => self.with_table(table, |t| t.insert(row))?, - Some(k) => self.with_indexed_table(table, |t| t.insert_keyed(row, *k))?, - } - let row: chusql_storage::protocol::Row = row.clone().into_iter().collect(); + WalOp::Insert { table, row } => { + self.with_table(table, |t| t.insert_row(row))?; let mut c = self.catalog.lock().unwrap(); - if c.record_insert(table, &row) { + if c.record_insert(table, row) { c.save(self.catalog_path())?; } Ok(()) } - WalOp::ReplaceAll { table, rows } => { - self.with_indexed_table(table, |t| t.replace_all(rows))?; - let rows: Vec = - rows.iter().map(|r| r.clone().into_iter().collect()).collect(); + WalOp::InsertBatch { table, rows } => { + self.with_table(table, |t| t.insert_rows(rows))?; let mut c = self.catalog.lock().unwrap(); - if c.record_replace_all(table, &rows) { + let mut changed = false; + for r in rows { + if c.record_insert(table, r) { + changed = true; + } + } + if changed { c.save(self.catalog_path())?; } Ok(()) } + WalOp::DeleteKeys { table, keys } => { + let deleted = self.with_table(table, |t| t.delete_by_keys(keys))?; + let mut c = self.catalog.lock().unwrap(); + c.record_delete(table, &deleted); + Ok(()) + } + WalOp::ReplaceAll { table, rows } => { + self.with_table(table, |t| t.replace_all(rows))?; + // 整表重写 = 统计与行数从头再数一遍 + let mut c = self.catalog.lock().unwrap(); + c.rebuild_stats(table, rows); + c.save(self.catalog_path()) + } } } @@ -171,9 +203,11 @@ impl Server { fn write_via_wal(&self, op: &WalOp) -> std::io::Result<()> { let _guard = self.write_lock.lock().unwrap(); std::fs::create_dir_all(&self.cfg.data_dir)?; - let wal = Wal::new(self.wal_path()); + let wal = &self.wal; - wal.clear()?; + // 只截断不落盘:上一条写操作最后已经把 WAL 刷成空的了, + // 这里的截断只是保证下面追加的是唯一一条,跟着的 append 会顺手把它落下去。 + wal.truncate()?; wal.append(op)?; @@ -185,30 +219,26 @@ impl Server { } } + // 这一步必须落盘:WAL 空 = 这条操作已经提交 wal.clear()?; result } /// 把涉及的表写回磁盘 fn flush_op_tables(&self, op: &WalOp) -> std::io::Result<()> { - match op { - WalOp::Insert { table, key, .. } => { - if key.is_some() { - self.with_indexed_table(table, |t| t.flush()) - } else { - self.with_table(table, |t| t.flush()) - } - } - WalOp::ReplaceAll { table, .. } => { - self.with_indexed_table(table, |t| t.flush()) - } - } + let table = match op { + WalOp::Insert { table, .. } + | WalOp::InsertBatch { table, .. } + | WalOp::DeleteKeys { table, .. } + | WalOp::ReplaceAll { table, .. } => table, + }; + self.with_table(table, |t| t.flush()) } /// 启动时重放残留 WAL fn recover_wal(&self) -> std::io::Result<()> { std::fs::create_dir_all(&self.cfg.data_dir)?; - let wal = Wal::new(self.wal_path()); + let wal = &self.wal; match wal.read()? { None => Ok(()), Some(op) => { @@ -221,8 +251,103 @@ impl Server { } } + /// 启动时把列统计重数一遍。 + /// + /// 统计是"每列见过哪些值"精确数出来的,只在内存里维护(随 catalog 落盘), + /// 所以重启之后要扫一遍表把它重建出来;否则统计会随重启慢慢失真。 + fn rebuild_stats(&self) -> std::io::Result<()> { + let names: Vec = self.catalog.lock().unwrap().table_names(); + let mut touched = false; + for table in names { + if !self.table_path(&table).exists() { + continue; + } + let rows = self.with_table(&table, |t| t.scan())?; + let mut c = self.catalog.lock().unwrap(); + c.rebuild_stats(&table, &rows); + touched = true; + } + if touched { + let c = self.catalog.lock().unwrap(); + c.save(self.catalog_path())?; + } + Ok(()) + } -// 请求分发 + // 索引维护 + /// 给一列建索引 + fn create_index(&self, table: &str, column: &str) -> std::io::Result<()> { + if column == "id" { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "id 是内建索引,不需要再建", + )); + } + { + let c = self.catalog.lock().unwrap(); + let Some(schema) = c.describe(table) else { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown table: {}", table), + )); + }; + if !schema.columns.iter().any(|c| c.name == column) { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown column: {}", column), + )); + } + if schema.indexes.iter().any(|i| i.column == column) { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("index on column \"{}\" already exists", column), + )); + } + } + + let path = self.index_path(table, column); + let (page_size, order, pool) = (self.cfg.page_size, self.cfg.btree_order, self.cfg.pool_size); + self.with_table(table, |t| { + t.build_index(column, &path, page_size, order, pool) + })?; + + let mut c = self.catalog.lock().unwrap(); + c.add_index(table, column)?; + c.save(self.catalog_path()) + } + + /// 去掉一列的索引(连文件一起删) + fn drop_index(&self, table: &str, column: &str) -> std::io::Result<()> { + if column == "id" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "id 是内建索引,不能删", + )); + } + { + let c = self.catalog.lock().unwrap(); + if c.describe(table).is_none() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown table: {}", table), + )); + } + } + self.with_table(table, |t| { + t.detach_index(column); + Ok(()) + })?; + + let mut c = self.catalog.lock().unwrap(); + c.remove_index(table, column)?; + c.save(self.catalog_path())?; + drop(c); + + let _ = std::fs::remove_file(self.index_path(table, column)); + Ok(()) + } + + // 请求分发 /// 一条请求翻成一条响应 fn handle_request(&self, req: Request) -> Response { match req { @@ -247,10 +372,10 @@ impl Server { } } - Request::Insert { table, row, key } => { - log_debug!(pipe, "insert table={} key={:?}", table, key); + Request::Insert { table, row } => { + log_debug!(pipe, "insert table={}", table); let tname = table.clone(); - let op = WalOp::Insert { table, row, key }; + let op = WalOp::Insert { table, row }; match self.write_via_wal(&op) { Ok(()) => Response::Ok, Err(e) => { @@ -262,9 +387,54 @@ impl Server { } } - Request::LookupByIndex { table, key } => { - log_debug!(pipe, "lookup_by_index table={} key={}", table, key); - match self.with_indexed_table(&table, |t| t.get_by_key(key)) { + Request::InsertBatch { table, rows } => { + log_debug!(pipe, "insert_batch table={} rows={}", table, rows.len()); + let tname = table.clone(); + let op = WalOp::InsertBatch { table, rows }; + match self.write_via_wal(&op) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "insert_batch {}: {}", tname, e); + Response::Error { + message: format!("insert_batch {}: {}", tname, e), + } + } + } + } + + Request::DeleteKeys { table, keys } => { + log_debug!(pipe, "delete_keys table={} keys={}", table, keys.len()); + let tname = table.clone(); + let op = WalOp::DeleteKeys { table, keys }; + match self.write_via_wal(&op) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "delete_keys {}: {}", tname, e); + Response::Error { + message: format!("delete_keys {}: {}", tname, e), + } + } + } + } + + Request::LookupByIndex { table, column, key } => { + log_debug!(pipe, "lookup_by_index table={} column={} key={}", table, column, key); + let indexed = + match self.with_existing_table(&table, |t| Ok(t.has_index(&column))) { + Ok(v) => v, + Err(e) => { + log_warn!(pipe, "lookup_by_index {}: {}", table, e); + return Response::Error { + message: format!("lookup_by_index {}: {}", table, e), + }; + } + }; + if !indexed { + // 没有这个索引:告诉查询层"这条我帮不上忙",让它退回全表扫描。 + // 回 NoIndex 而不是空结果,是因为"没有索引"和"没查到"必须分开。 + return Response::NoIndex; + } + match self.with_table(&table, |t| t.get_by_column_key(&column, key)) { Ok(Some(row)) => Response::Rows { rows: vec![row] }, Ok(None) => Response::Rows { rows: vec![] }, Err(e) => { @@ -314,6 +484,8 @@ impl Server { table: name.to_string(), columns: ts.columns.clone(), row_count: ts.row_count, + indexes: index_wires(ts), + stats: stat_wires(ts), }) .collect(); Response::Catalog { schemas } @@ -322,9 +494,12 @@ impl Server { Request::DescribeTable { table } => { log_debug!(pipe, "describe_table table={}", table); match self.describe_table(&table) { - Ok((cols, count)) => Response::Schema { + Ok((cols, count, indexes, stats)) => Response::Schema { + table, columns: cols, row_count: count, + indexes, + stats, }, Err(e) => { log_warn!(pipe, "describe_table {}: {}", table, e); @@ -370,8 +545,12 @@ impl Server { Request::DropTable { table } => { log_debug!(pipe, "drop_table table={}", table); - { + let dropped_indexes: Vec = { let mut c = self.catalog.lock().unwrap(); + let idx = c + .describe(&table) + .map(|s| s.indexes.iter().map(|i| i.column.clone()).collect()) + .unwrap_or_default(); if let Err(e) = c.drop_table(&table) { return Response::Error { message: format!("drop_table {}: {}", table, e), @@ -382,18 +561,48 @@ impl Server { message: format!("drop_table {}: catalog save: {}", table, e), }; } - } + idx + }; self.tables.lock().unwrap().remove(&table); let _ = std::fs::remove_file(self.table_path(&table)); - let _ = std::fs::remove_file(self.index_path(&table)); + let _ = std::fs::remove_file(self.index_path(&table, "id")); + for col in dropped_indexes { + let _ = std::fs::remove_file(self.index_path(&table, &col)); + } Response::Ok } + + Request::CreateIndex { table, column } => { + log_debug!(pipe, "create_index table={} column={}", table, column); + match self.create_index(&table, &column) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "create_index {}({}): {}", table, column, e); + Response::Error { + message: format!("create_index {}({}): {}", table, column, e), + } + } + } + } + + Request::DropIndex { table, column } => { + log_debug!(pipe, "drop_index table={} column={}", table, column); + match self.drop_index(&table, &column) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "drop_index {}({}): {}", table, column, e); + Response::Error { + message: format!("drop_index {}({}): {}", table, column, e), + } + } + } + } } } -// 信息查询 + // 信息查询 /// 表名:catalog 与目录取并集 fn list_tables(&self) -> std::io::Result> { let mut names = BTreeSet::new(); @@ -416,15 +625,23 @@ impl Server { Ok(names.into_iter().collect()) } - /// 查一张表的 schema - fn describe_table(&self, table: &str) -> std::io::Result<(Vec, u64)> { + /// 查一张表的 schema(列 + 行数 + 索引 + 统计) + fn describe_table( + &self, + table: &str, + ) -> std::io::Result<(Vec, u64, Vec, Vec)> { let c = self.catalog.lock().unwrap(); if let Some(schema) = c.describe(table) { - return Ok((schema.columns.clone(), schema.row_count)); + return Ok(( + schema.columns.clone(), + schema.row_count, + index_wires(schema), + stat_wires(schema), + )); } drop(c); if std::fs::metadata(self.table_path(table)).is_ok() { - return Ok((Vec::new(), 0)); + return Ok((Vec::new(), 0, Vec::new(), Vec::new())); } Err(std::io::Error::new( std::io::ErrorKind::NotFound, @@ -433,6 +650,35 @@ impl Server { } } +/// 一张表的索引定义 -> 线上格式。 +/// `id` 是内建索引(表一开出来就有),数据字典里不记它,但对外要报出来。 +fn index_wires(schema: &chusql_storage::catalog::TableSchema) -> Vec { + let mut out = vec![IndexWire { + column: "id".to_string(), + }]; + for i in &schema.indexes { + if i.column != "id" { + out.push(IndexWire { + column: i.column.clone(), + }); + } + } + out +} + +/// 一张表的列统计 -> 线上格式 +fn stat_wires(schema: &chusql_storage::catalog::TableSchema) -> Vec { + schema + .stats + .iter() + .map(|(name, s)| ColumnStatWire { + name: name.clone(), + distinct: s.distinct, + capped: s.capped, + }) + .collect() +} + impl Drop for Server { fn drop(&mut self) { if let Ok(c) = self.catalog.lock() { @@ -441,7 +687,6 @@ impl Drop for Server { } } - // 连接 /// 处理一条连接 fn handle(server: &Server, conn: LocalSocketStream) -> std::io::Result<()> { @@ -471,7 +716,6 @@ fn handle(server: &Server, conn: LocalSocketStream) -> std::io::Result<()> { Ok(()) } - // 入口 /// 把配置来源打进日志 fn log_config(loaded: &Loaded) { @@ -506,6 +750,11 @@ fn main() -> std::io::Result<()> { return Err(e); } + if let Err(e) = server.rebuild_stats() { + // 统计重建失败不影响服务:报一声,继续跑(统计会停留在落盘时的值) + log_warn!(core, "stats rebuild skipped: {}", e); + } + let ns_name = server .cfg .pipe_name diff --git a/chusql-storage/src/btree.rs b/chusql-storage/src/btree.rs index a9e06b5..995e953 100644 --- a/chusql-storage/src/btree.rs +++ b/chusql-storage/src/btree.rs @@ -430,6 +430,41 @@ impl DiskBTree { } } + /// 删掉一个键;键不存在返回 false。 + /// + /// 这是**惰性删除**:只把叶子里那一对 (key, value) 拿掉,不做合并、也不向兄弟借位。 + /// 于是树可能变稀、甚至留下空叶子,但查找依然正确——内部节点的分隔键仍然满足 + /// "小于等于它的往左、大于它的往右",空叶子二分找不到东西自然返回 None。 + /// 代价是空间不回收(节点删空了也不还页),整表重写(`clear`)时会一起重建。 + pub fn delete(&mut self, key: i64) -> io::Result { + match self.root()? { + None => Ok(false), + Some(r) => self.delete_rec(r, key), + } + } + + /// 递归删除:一路走到叶子,删掉那一对 + fn delete_rec(&mut self, page_id: PageId, key: i64) -> io::Result { + let order = self.order; + match self.file.with_page(page_id, |p| read_node(p, order))? { + NodeData::Leaf { mut keys, mut values, next } => match keys.binary_search(&key) { + Err(_) => Ok(false), + Ok(i) => { + keys.remove(i); + values.remove(i); + self.file.update_page(page_id, |p| { + write_node(p, &NodeData::Leaf { keys, values, next }, order) + })?; + Ok(true) + } + }, + NodeData::Internal { keys, children } => { + let i = child_index(&keys, key); + self.delete_rec(children[i], key) + } + } + } + /// 按键升序返回全部 pub fn iter_all(&mut self) -> io::Result> { let mut out = Vec::new(); diff --git a/chusql-storage/src/catalog.rs b/chusql-storage/src/catalog.rs index 076a403..250393d 100644 --- a/chusql-storage/src/catalog.rs +++ b/chusql-storage/src/catalog.rs @@ -1,5 +1,7 @@ -use std::collections::BTreeMap; +use std::collections::hash_map::DefaultHasher; +use std::collections::{BTreeMap, HashSet}; +use std::hash::{Hash, Hasher}; use std::io; use std::path::Path; @@ -7,24 +9,52 @@ use serde::{Deserialize, Serialize}; use crate::protocol::{ColumnType, Row, SchemaColumn}; -// 数据字典:记每张表有哪些列、什么类型、多少行。 +// 数据字典:记每张表有哪些列、什么类型、多少行、有哪些索引,以及列级统计。 +// +// 统计只在内存里精确维护(每列见过哪些值的哈希,有上限),随 catalog.json 一起落盘; +// 服务启动时会扫一遍表把统计重建出来,所以重启之后数字仍然是对的。 + +/// 一列的不同值个数最多精确统计到这里;超过就只报"下界" +const STATS_DISTINCT_CAP: u64 = 4096; + +// 表结构 +#[derive(Debug, Clone, Serialize, Deserialize)] +/// 一条索引定义:只记"盯的是哪一列"(索引按列命名,一列最多一个) +pub struct IndexSchema { + pub column: String, +} #[derive(Debug, Clone, Default, Serialize, Deserialize)] -/// 一张表的 schema +/// 一列的统计 +pub struct ColumnStat { + /// 不同值个数(`capped` 为真时是下界) + pub distinct: u64, + /// 不同值太多,已经不再精确统计 + #[serde(default)] + pub capped: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +/// 一张表的 schema + 统计 pub struct TableSchema { pub columns: Vec, #[serde(default)] pub row_count: u64, + #[serde(default)] + pub indexes: Vec, + #[serde(default)] + pub stats: BTreeMap, } #[derive(Debug, Default, Serialize, Deserialize)] /// 数据字典:表名到 schema pub struct Catalog { tables: BTreeMap, + /// 每列"见过的值"的哈希集合(只在内存里,不落盘;有上限) + #[serde(skip)] + seen: BTreeMap>, } - - impl Catalog { // 读取与保存 /// 从文件读;不存在给空字典 @@ -65,17 +95,7 @@ impl Catalog { /// 按一行数据补列 pub fn ensure_columns(&mut self, table: &str, row: &Row) { let entry = self.tables.entry(table.to_string()).or_default(); - for (k, v) in row { - if entry.columns.iter().any(|c| &c.name == k) { - continue; - } - if let Some(ty) = infer_type(v) { - entry.columns.push(SchemaColumn { - name: k.clone(), - ty, - }); - } - } + add_missing_columns(entry, row); } /// 建表;已存在报错 @@ -91,67 +111,144 @@ impl Catalog { TableSchema { columns, row_count: 0, + indexes: Vec::new(), + stats: BTreeMap::new(), }, ); Ok(()) } - /// 按多行数据补列 - pub fn ensure_columns_from_rows(&mut self, table: &str, rows: &[Row]) { - for r in rows { - self.ensure_columns(table, r); - } - } - /// 记一次插入:补列 + 行数 +1 + /// 记一次插入:补列 + 行数 +1 + 更新列统计;返回"列有没有变化"(决定要不要立刻写盘) pub fn record_insert(&mut self, table: &str, row: &Row) -> bool { - let entry = self.tables.entry(table.to_string()).or_default(); - let mut changed = false; - for (k, v) in row { - if entry.columns.iter().any(|c| &c.name == k) { - continue; - } - if let Some(ty) = infer_type(v) { - entry.columns.push(SchemaColumn { name: k.clone(), ty }); - changed = true; - } - } - entry.row_count += 1; + let changed = { + let entry = self.tables.entry(table.to_string()).or_default(); + let changed = add_missing_columns(entry, row); + entry.row_count += 1; + changed + }; + self.count_values(table, row); changed } - - /// 记一次整表替换:列照补,行数重置 - pub fn record_replace_all(&mut self, table: &str, rows: &[Row]) -> bool { + /// 记一次删除:行数减少 + 更新列统计 + pub fn record_delete(&mut self, table: &str, rows: &[Row]) { let entry = self.tables.entry(table.to_string()).or_default(); - let mut changed = false; + entry.row_count = entry.row_count.saturating_sub(rows.len() as u64); + let seen = self.seen.entry(table.to_string()).or_default(); for r in rows { - for (k, v) in r { - if entry.columns.iter().any(|c| &c.name == k) { + for (name, v) in r { + let stat = entry.stats.entry(name.clone()).or_default(); + if stat.capped { continue; } - if let Some(ty) = infer_type(v) { - entry.columns.push(SchemaColumn { name: k.clone(), ty }); - changed = true; + // 这个值彻底没了才减:集合里删得掉说明没有别的行还用它 + if seen.remove(&fold(name, value_hash(v))) { + stat.distinct = stat.distinct.saturating_sub(1); + } + } + } + } + + /// 重数一遍统计与行数(启动时重建、整表改写完都走它);返回"列有没有变化" + pub fn rebuild_stats(&mut self, table: &str, rows: &[Row]) -> bool { + self.seen.remove(table); + { + let entry = self.tables.entry(table.to_string()).or_default(); + entry.stats.clear(); + entry.row_count = 0; + } + // 逐行"插入"地数:行数、列、统计一次到位 + rows.iter() + .fold(false, |changed, r| self.record_insert(table, r) || changed) + } + + /// 列统计:这一列第一次见到这个值就 distinct +1(到上限之后只报下界) + fn count_values(&mut self, table: &str, row: &Row) { + let entry = self.tables.entry(table.to_string()).or_default(); + let seen = self.seen.entry(table.to_string()).or_default(); + for (name, v) in row { + let stat = entry.stats.entry(name.clone()).or_default(); + if stat.capped { + continue; + } + if seen.insert(fold(name, value_hash(v))) { + stat.distinct += 1; + if stat.distinct >= STATS_DISTINCT_CAP { + stat.capped = true; + seen.clear(); // 到上限就不再精确统计,把内存还回去 } } } - entry.row_count = rows.len() as u64; - changed } /// 删表 - pub fn drop_table(&mut self, table: &str) -> io::Result<()> { - if self.tables.remove(table).is_none() { + pub fn drop_table(&mut self, table: &str) -> io::Result<()> { if self.tables.remove(table).is_none() { return Err(io::Error::new( io::ErrorKind::NotFound, format!("unknown table: {}", table), )); } + self.seen.remove(table); Ok(()) } + + // 索引 + /// 记下一条索引定义 + pub fn add_index(&mut self, table: &str, column: &str) -> io::Result<()> { + let entry = self + .tables + .get_mut(table) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?; + if entry.indexes.iter().any(|i| i.column == column) { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("index on column \"{}\" already exists", column), + )); + } + entry.indexes.push(IndexSchema { + column: column.to_string(), + }); + Ok(()) + } + + /// 去掉一条索引定义 + pub fn remove_index(&mut self, table: &str, column: &str) -> io::Result<()> { + let entry = self + .tables + .get_mut(table) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("unknown table: {}", table)))?; + match entry.indexes.iter().position(|i| i.column == column) { + None => Err(io::Error::new( + io::ErrorKind::NotFound, + format!("no index on column \"{}\"", column), + )), + Some(i) => { + entry.indexes.remove(i); + Ok(()) + } + } + } } // 推断 +/// 把行里"字典还不知道的列"补进来;返回有没有变化 +fn add_missing_columns(entry: &mut TableSchema, row: &Row) -> bool { + let mut changed = false; + for (k, v) in row { + if entry.columns.iter().any(|c| &c.name == k) { + continue; + } + if let Some(ty) = infer_type(v) { + entry.columns.push(SchemaColumn { + name: k.clone(), + ty, + }); + changed = true; + } + } + changed +} + /// 由 JSON 值猜列类型 fn infer_type(v: &serde_json::Value) -> Option { match v { @@ -161,3 +258,18 @@ fn infer_type(v: &serde_json::Value) -> Option { _ => None, } } + +/// 一个值的哈希(按它的 JSON 文本算;统计"不同值个数"用) +fn value_hash(v: &serde_json::Value) -> u64 { + let mut h = DefaultHasher::new(); + v.to_string().hash(&mut h); + h.finish() +} + +/// 把"列名"和"值哈希"合成一个哈希:避免两列的同值互相干扰 +fn fold(column: &str, value_hash: u64) -> u64 { + let mut h = DefaultHasher::new(); + column.hash(&mut h); + value_hash.hash(&mut h); + h.finish() +} diff --git a/chusql-storage/src/heap.rs b/chusql-storage/src/heap.rs index f7cc6df..c19e599 100644 --- a/chusql-storage/src/heap.rs +++ b/chusql-storage/src/heap.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::io; use std::path::Path; @@ -6,7 +7,10 @@ use crate::page::PageId; use crate::page::{Page, PageFile}; use crate::protocol::Row; -// 堆表:行按插入顺序进页,支持全表扫描与按索引取行。 +// 堆表:行按插入顺序进页,支持全表扫描、按索引取行、行级删除。 +// +// 索引由**表自己维护**:有哪些索引记在数据字典里,插入时按索引的列从行里取值写进去, +// 调用方不需要知道有哪些索引,也不需要自己指定 key。 // 页内布局 /// 页头:slot_count 占 2 字节 @@ -47,6 +51,14 @@ fn unpack_position(v: u64) -> (PageId, u16) { (v >> 16, (v & 0xFFFF) as u16) } +/// 取行里某一列的整数值(不是整数就没有) +fn row_int(row: &Row, column: &str) -> Option { + match row.get(column) { + Some(serde_json::Value::Number(n)) => n.as_i64(), + _ => None, + } +} + // 页内操作 /// 只管一页里的字节 pub struct HeapPage; @@ -103,6 +115,20 @@ impl HeapPage { Some(n) } + /// 删掉一个槽位:只把长度置 0,行字节原地留着(读的时候按"长度 0 = 没有这一行"处理) + pub fn delete_tuple(page: &mut Page, slot: u16) -> bool { + let n = Self::slot_count(page); + if slot >= n { + return false; + } + let slot_off = SLOT_DIR_START + (slot as usize) * SLOT_SIZE; + if read_u32(&page.data, slot_off + 4) == 0 { + return false; + } + write_u32(&mut page.data, slot_off + 4, 0); + true + } + /// 读槽位里的行字节 pub fn get_tuple(page: &Page, slot: u16) -> Option> { let n = Self::slot_count(page); @@ -124,27 +150,30 @@ impl HeapPage { } } +// 索引 +/// 表上的一个索引:盯着一列,键是这一列的整数值 +struct NamedIndex { + column: String, + tree: DiskBTree, +} + // 堆表 -/// 一个文件 = 若干页 +/// 一个数据文件 = 若干页;外加若干索引文件 pub struct HeapTable { file: PageFile, - index: Option, + indexes: Vec, } impl HeapTable { /// 打开表(不带索引) - pub fn open>( - path: P, - page_size: usize, - pool_size: usize, - ) -> io::Result { - Ok(HeapTable { + pub fn open>(path: P, page_size: usize, pool_size: usize) -> io::Result { + Ok(HeapTable { file: PageFile::with_options(path, page_size, pool_size)?, - index: None, + indexes: Vec::new(), }) } - /// 打开表 + 索引 + /// 打开表 + `id` 列索引(`id` 沿用老文件名 `表名.idx`) pub fn open_indexed, Q: AsRef>( path: P, index_path: Q, @@ -152,19 +181,155 @@ impl HeapTable { btree_order: usize, pool_size: usize, ) -> io::Result { - Ok(HeapTable { + let mut t = HeapTable { file: PageFile::with_options(path, page_size, pool_size)?, - index: Some(DiskBTree::open(index_path, page_size, btree_order, pool_size)?), - }) + indexes: Vec::new(), + }; + t.attach_index("id", index_path, page_size, btree_order, pool_size)?; + Ok(t) } - /// 插入一行,不更新索引 - pub fn insert(&mut self, row: &Row) -> io::Result<()> { - self.insert_returning_position(row).map(|_| ()) + /// 打开表 + 任意组索引:(列名, 索引文件路径) + pub fn open_with_indexes>( + path: P, + indexes: &[(String, std::path::PathBuf)], + page_size: usize, + btree_order: usize, + pool_size: usize, + ) -> io::Result { + let mut t = HeapTable { + file: PageFile::with_options(path, page_size, pool_size)?, + indexes: Vec::new(), + }; + for (col, p) in indexes { + t.attach_index(col, p, page_size, btree_order, pool_size)?; + } + Ok(t) } - /// 插入并返回行位置 - pub fn insert_returning_position(&mut self, row: &Row) -> io::Result<(PageId, u16)> { + /// 挂上一棵索引树(只打开/创建文件,不填数据) + fn attach_index>( + &mut self, + column: &str, + path: P, + page_size: usize, + btree_order: usize, + pool_size: usize, + ) -> io::Result<()> { + let tree = DiskBTree::open(path, page_size, btree_order, pool_size)?; + self.indexes.push(NamedIndex { + column: column.to_string(), + tree, + }); + Ok(()) + } + + /// 这一列有索引吗 + pub fn has_index(&self, column: &str) -> bool { + self.indexes.iter().any(|i| i.column == column) + } + + /// 建索引:先扫一遍查重,再建树填数据。 + /// + /// **要求这一列的整数取值唯一**,不唯一就报错、一个索引文件都不留。 + /// 这样"走索引取一行"和"全表扫一遍再筛"必然得到同一个答案; + /// 非唯一索引要等 B+ 树支持重复键(记在项目计划的 P2-② 里)。 + pub fn build_index( + &mut self, + column: &str, + path: &Path, + page_size: usize, + btree_order: usize, + pool_size: usize, + ) -> io::Result<()> { + if self.has_index(column) { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("index on column \"{}\" already exists", column), + )); + } + + // 第 1 步:扫全表,查重并攒下 (键, 位置) + let mut seen: HashSet = HashSet::new(); + let mut entries: Vec<(i64, u64)> = Vec::new(); + for (page_id, slot, r) in self.scan_with_positions()? { + let Some(k) = row_int(&r, column) else { + continue; // 这一列不是整数的行不进索引 + }; + if !seen.insert(k) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "column \"{}\" has duplicate value {} — 暂不支持非唯一索引", + column, k + ), + )); + } + entries.push((k, pack_position(page_id, slot))); + } + + // 第 2 步:建树 + self.attach_index(column, path, page_size, btree_order, pool_size)?; + let i = self.indexes.len() - 1; + self.indexes[i].tree.clear()?; // 清掉可能残留的旧文件内容 + for (k, pos) in entries { + self.indexes[i].tree.insert(k, pos)?; + } + Ok(()) + } + + /// 摘掉一个索引(不删文件) + pub fn detach_index(&mut self, column: &str) -> bool { + match self.indexes.iter().position(|i| i.column == column) { + None => false, + Some(i) => { + self.indexes.remove(i); + true + } + } + } + + /// 插入一行:进堆表,并按每个索引的列顺手写索引 + pub fn insert_row(&mut self, row: &Row) -> io::Result<()> { + self.insert_row_returning_position(row).map(|_| ()) + } + + /// 插入一行并返回它的位置 + pub fn insert_row_returning_position(&mut self, row: &Row) -> io::Result<(PageId, u16)> { + // 先把键算好并查重:**查重必须在落堆之前**, + // 否则某个索引写不进去时行已经进堆了,表和索引就不一致了。 + let mut pending: Vec<(usize, i64)> = Vec::new(); + for (i, idx) in self.indexes.iter_mut().enumerate() { + let Some(k) = row_int(row, &idx.column) else { + continue; // 这一列不是整数(或没这一列):这行不进这个索引 + }; + if idx.tree.get(k)?.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("duplicate key {} on indexed column \"{}\"", k, idx.column), + )); + } + pending.push((i, k)); + } + + let (page_id, slot) = self.insert_returning_position(row)?; + let pos = pack_position(page_id, slot); + for (i, k) in pending { + self.indexes[i].tree.insert(k, pos)?; + } + Ok((page_id, slot)) + } + + /// 批量插入(只写缓冲池,落盘由调用方 `flush` 一次完成) + pub fn insert_rows(&mut self, rows: &[Row]) -> io::Result<()> { + for r in rows { + self.insert_row(r)?; + } + Ok(()) + } + + /// 插入一行,只进堆表、不碰索引 + fn insert_returning_position(&mut self, row: &Row) -> io::Result<(PageId, u16)> { let tuple = serde_json::to_vec(row).map_err(io::Error::other)?; if HEADER_SIZE + SLOT_SIZE + tuple.len() > self.file.page_size() { @@ -193,23 +358,18 @@ impl HeapTable { Ok((page_id, slot)) } - /// 插入一行并写索引 - pub fn insert_keyed(&mut self, row: &Row, key: i64) -> io::Result<()> { - let (page_id, slot) = self.insert_returning_position(row)?; - if let Some(idx) = &mut self.index { - idx.insert(key, pack_position(page_id, slot))?; - } - Ok(()) + /// 按 `id` 取一行 + pub fn get_by_key(&mut self, key: i64) -> io::Result> { + self.get_by_column_key("id", key) } - /// 按索引键取一行 - pub fn get_by_key(&mut self, key: i64) -> io::Result> { - let pos = match &mut self.index { - None => return Ok(None), - Some(idx) => match idx.get(key)? { - None => return Ok(None), - Some(p) => p, - }, + /// 按某一列的索引取一行 + pub fn get_by_column_key(&mut self, column: &str, key: i64) -> io::Result> { + let Some(idx) = self.indexes.iter_mut().find(|i| i.column == column) else { + return Ok(None); + }; + let Some(pos) = idx.tree.get(key)? else { + return Ok(None); }; let (page_id, slot) = unpack_position(pos); self.read_at(page_id, slot) @@ -227,47 +387,115 @@ impl HeapTable { } } - /// 全表扫描 - pub fn scan(&mut self) -> io::Result> { - let n = self.file.num_pages()?; - let mut rows = Vec::new(); - for i in 0..n { - let tuples = self.file.with_page(i, |p| { - let mut out = Vec::new(); - for slot in HeapPage::iter_slots(p) { - if let Some(bytes) = HeapPage::get_tuple(p, slot) { - out.push(bytes); - } + /// 按位置删一行:槽位长度置 0,从所有索引里摘掉,并把这一行交出来 + pub fn delete_at(&mut self, page_id: PageId, slot: u16) -> io::Result> { + // 先读出来:删完就不知道这行各列是什么,索引也就无从摘起 + let row = self.read_at(page_id, slot)?; + let existed = self.file.update_page(page_id, |p| HeapPage::delete_tuple(p, slot))?; + if !existed { + return Ok(None); + } + if let Some(r) = &row { + for idx in &mut self.indexes { + if let Some(k) = row_int(r, &idx.column) { + idx.tree.delete(k)?; } - out - })?; - for bytes in tuples { - let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?; - rows.push(row); } } - Ok(rows) + Ok(row) } - /// 整表改写并重建索引 + /// 按 `id` 批量删行,返回被删掉的那些行(数据字典要用它更新统计)。 + /// + /// 定位靠 `id` 索引;没有这个索引就退化成一趟全表扫描(仍然只删目标行)。 + pub fn delete_by_keys(&mut self, keys: &[i64]) -> io::Result> { + let mut positions: Vec<(PageId, u16)> = Vec::new(); + + if self.has_index("id") { + for &k in keys { + let pos = self + .indexes + .iter_mut() + .find(|i| i.column == "id") + .and_then(|i| i.tree.get(k).ok().flatten()); + if let Some(p) = pos { + positions.push(unpack_position(p)); + } + } + } else { + let mut wanted: Vec = keys.to_vec(); + wanted.sort_unstable(); + for (page_id, slot, r) in self.scan_with_positions()? { + if let Some(k) = row_int(&r, "id") { + if wanted.binary_search(&k).is_ok() { + positions.push((page_id, slot)); + } + } + } + } + + let mut deleted: Vec = Vec::new(); + for (page_id, slot) in positions { + if let Some(r) = self.delete_at(page_id, slot)? { + deleted.push(r); + } + } + Ok(deleted) + } + + /// 全表扫描 + pub fn scan(&mut self) -> io::Result> { + Ok(self + .scan_with_positions()? + .into_iter() + .map(|(_, _, r)| r) + .collect()) + } + + /// 全表扫描,同时给出每行的位置(建索引用) + pub fn scan_with_positions(&mut self) -> io::Result> { + let n = self.file.num_pages()?; + let mut out = Vec::new(); + for i in 0..n { + let tuples = self.file.with_page(i, |p| { + let mut v = Vec::new(); + for slot in HeapPage::iter_slots(p) { + if let Some(bytes) = HeapPage::get_tuple(p, slot) { + v.push((slot, bytes)); + } + } + v + })?; + for (slot, bytes) in tuples { + let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?; + out.push((i, slot, row)); + } + } + Ok(out) + } + + /// 整表改写并重建所有索引 pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> { self.file.truncate()?; - if let Some(idx) = &mut self.index { - idx.clear()?; + for idx in &mut self.indexes { + idx.tree.clear()?; } for r in rows { let (page_id, slot) = self.insert_returning_position(r)?; - if let (Some(idx), Some(key)) = (&mut self.index, row_key(r)) { - idx.insert(key, pack_position(page_id, slot))?; + let pos = pack_position(page_id, slot); + for idx in &mut self.indexes { + if let Some(k) = row_int(r, &idx.column) { + idx.tree.insert(k, pos)?; + } } } Ok(()) } - /// 把脏页写回 + /// 把脏页写回(索引也一起) pub fn flush(&mut self) -> io::Result<()> { - if let Some(idx) = &mut self.index { - idx.flush()?; + for idx in &mut self.indexes { + idx.tree.flush()?; } self.file.flush() } @@ -282,11 +510,3 @@ impl HeapTable { /// 命中率 pub fn hit_rate(&self) -> f64 { self.file.hit_rate() } } - -/// 取行里的 id 作索引键 -fn row_key(row: &Row) -> Option { - match row.get("id") { - Some(serde_json::Value::Number(n)) => n.as_i64(), - _ => None, - } -} diff --git a/chusql-storage/src/protocol.rs b/chusql-storage/src/protocol.rs index 7af16eb..d1163f8 100644 --- a/chusql-storage/src/protocol.rs +++ b/chusql-storage/src/protocol.rs @@ -22,42 +22,104 @@ pub struct SchemaColumn { pub ty: ColumnType, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// 一列的统计(随数据字典一起走) +pub struct ColumnStatWire { + pub name: String, + /// 不同值个数;`capped` 为真时它只是下界 + pub distinct: u64, + #[serde(default)] + pub capped: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// 一条索引定义 +pub struct IndexWire { + pub column: String, +} + #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "method", rename_all = "snake_case")] /// 请求:method 字段决定类型 pub enum Request { Ping, - Scan { table: String }, + Scan { + table: String, + }, + /// 插一行;要不要写索引由表上的索引决定(调用方不用管) Insert { table: String, row: Row, - #[serde(default)] - key: Option, }, - LookupByIndex { table: String, key: i64 }, - ReplaceAll { table: String, rows: Vec }, + /// 一批行一起插:一次 WAL、一次 fsync + InsertBatch { + table: String, + rows: Vec, + }, + /// 按 id 批量删行 + DeleteKeys { + table: String, + keys: Vec, + }, + /// 按某一列的索引取一行(列名默认 "id",兼容老客户端) + LookupByIndex { + table: String, + #[serde(default = "default_id_column")] + column: String, + key: i64, + }, + ReplaceAll { + table: String, + rows: Vec, + }, ListTables, - DescribeTable { table: String }, + DescribeTable { + table: String, + }, CreateTable { table: String, columns: Vec, }, - DropTable { table: String }, - ListCatalog + DropTable { + table: String, + }, + ListCatalog, + /// 给某一列建索引(要求这一列整数取值唯一) + CreateIndex { + table: String, + column: String, + }, + /// 去掉某一列的索引 + DropIndex { + table: String, + column: String, + }, +} + +/// 没写 column 时按 `id` 算 +fn default_id_column() -> String { + "id".to_string() } #[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "status", rename_all = "lowercase")] +#[serde(tag = "status", rename_all = "snake_case")] /// 响应:status 字段决定类型 pub enum Response { Pong, Rows { rows: Vec }, Tables { tables: Vec }, Schema { + table: String, columns: Vec, #[serde(default)] row_count: u64, + #[serde(default)] + indexes: Vec, + #[serde(default)] + stats: Vec, }, + /// 这一列没有索引 —— 调用方应该退回全表扫描 + NoIndex, Ok, Error { message: String }, Catalog { schemas: Vec }, @@ -69,4 +131,8 @@ pub struct TableSchemaWire { pub columns: Vec, #[serde(default)] pub row_count: u64, + #[serde(default)] + pub indexes: Vec, + #[serde(default)] + pub stats: Vec, } diff --git a/chusql-storage/src/wal.rs b/chusql-storage/src/wal.rs index 2b2afab..b0dadb7 100644 --- a/chusql-storage/src/wal.rs +++ b/chusql-storage/src/wal.rs @@ -1,7 +1,8 @@ use std::fs::{File, OpenOptions}; -use std::io::{self, Read, Write}; +use std::io::{self, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::sync::Mutex; use serde::{Deserialize, Serialize}; @@ -11,6 +12,8 @@ use crate::protocol::Row; const OP_INSERT: u8 = 1; const OP_REPLACE_ALL: u8 = 2; +const OP_INSERT_BATCH: u8 = 3; +const OP_DELETE_KEYS: u8 = 4; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] /// 一条待写入的操作 @@ -18,7 +21,16 @@ pub enum WalOp { Insert { table: String, row: Row, - key: Option, + }, + /// 一批行:一次日志、一次 fsync 就能全部落盘 + InsertBatch { + table: String, + rows: Vec, + }, + /// 按 id 批量删行(重放是幂等的:再删一次等于什么都没删) + DeleteKeys { + table: String, + keys: Vec, }, ReplaceAll { table: String, @@ -27,8 +39,13 @@ pub enum WalOp { } /// WAL 文件 +/// +/// 句柄只开一次并一直留着:写入路径是每条操作一次 +/// `truncate -> append -> truncate`,每次都重新 open/close 文件要多花 +/// 一倍的时间(见 benchmark/chusql-storage)。 pub struct Wal { path: PathBuf, + handle: Mutex>, } impl Wal { @@ -37,6 +54,7 @@ impl Wal { pub fn new>(path: P) -> Self { Wal { path: path.as_ref().to_path_buf(), + handle: Mutex::new(None), } } @@ -45,26 +63,43 @@ impl Wal { &self.path } - /// 清空 WAL - pub fn clear(&self) -> io::Result<()> { - if self.path.exists() { - let f = OpenOptions::new().write(true).open(&self.path)?; - f.set_len(0)?; - f.sync_all()?; + /// 拿常驻句柄(没有就开一个) + fn with_handle(&self, f: impl FnOnce(&mut File) -> io::Result) -> io::Result { + let mut slot = self.handle.lock().unwrap(); + if slot.is_none() { + *slot = Some( + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&self.path)?, + ); } - Ok(()) + f(slot.as_mut().expect("just opened")) + } + + /// 截断到空并落盘 + pub fn clear(&self) -> io::Result<()> { + self.with_handle(|f| { + truncate(f)?; + f.sync_all() + }) + } + + /// 只截断、不落盘:紧跟其后的 `append` 会 fsync, + /// 那一次会把截断一起落下去,所以这里不必单独再刷一遍。 + pub fn truncate(&self) -> io::Result<()> { + self.with_handle(truncate) } /// 追加一条并 fsync pub fn append(&self, op: &WalOp) -> io::Result<()> { let bytes = encode(op)?; - let mut f = OpenOptions::new() - .create(true) - .append(true) - .open(&self.path)?; - f.write_all(&bytes)?; - f.sync_data()?; - Ok(()) + self.with_handle(|f| { + f.write_all(&bytes)?; + f.sync_data() + }) } /// 读当前 WAL;空则 None @@ -88,14 +123,29 @@ impl Wal { } } +/// 清空文件并回到起点(句柄不是 append 模式,位置要自己摆正) +fn truncate(f: &mut File) -> io::Result<()> { + f.set_len(0)?; + f.seek(SeekFrom::Start(0))?; + Ok(()) +} + // 编解码 /// 把操作编成字节 fn encode(op: &WalOp) -> io::Result> { let (op_type, table, payload) = match op { - WalOp::Insert { table, row, key } => { - let p = serde_json::to_vec(&(row, key)).map_err(io::Error::other)?; + WalOp::Insert { table, row } => { + let p = serde_json::to_vec(row).map_err(io::Error::other)?; (OP_INSERT, table.clone(), p) } + WalOp::InsertBatch { table, rows } => { + let p = serde_json::to_vec(rows).map_err(io::Error::other)?; + (OP_INSERT_BATCH, table.clone(), p) + } + WalOp::DeleteKeys { table, keys } => { + let p = serde_json::to_vec(keys).map_err(io::Error::other)?; + (OP_DELETE_KEYS, table.clone(), p) + } WalOp::ReplaceAll { table, rows } => { let p = serde_json::to_vec(rows).map_err(io::Error::other)?; (OP_REPLACE_ALL, table.clone(), p) @@ -145,9 +195,19 @@ fn decode(buf: &[u8]) -> Result, String> { let op = match op_type { OP_INSERT => { - let (row, key): (Row, Option) = serde_json::from_slice(payload) - .map_err(|e| format!("bad insert payload: {}", e))?; - WalOp::Insert { table, row, key } + let row: Row = + serde_json::from_slice(payload).map_err(|e| format!("bad insert payload: {}", e))?; + WalOp::Insert { table, row } + } + OP_INSERT_BATCH => { + let rows: Vec = serde_json::from_slice(payload) + .map_err(|e| format!("bad insert_batch payload: {}", e))?; + WalOp::InsertBatch { table, rows } + } + OP_DELETE_KEYS => { + let keys: Vec = serde_json::from_slice(payload) + .map_err(|e| format!("bad delete_keys payload: {}", e))?; + WalOp::DeleteKeys { table, keys } } OP_REPLACE_ALL => { let rows: Vec = serde_json::from_slice(payload) diff --git a/chusql-storage/tests/btree_test.rs b/chusql-storage/tests/btree_test.rs index 38442f9..5dcfddc 100644 --- a/chusql-storage/tests/btree_test.rs +++ b/chusql-storage/tests/btree_test.rs @@ -153,3 +153,63 @@ fn order_too_large_for_page_errors() { let dir = tempfile::tempdir().unwrap(); assert!(DiskBTree::open(dir.path().join("t.idx"), 512, 300, DEFAULT_POOL_SIZE).is_err()); } + +/// 删掉键之后查不到,别人不受影响 +#[test] +fn delete_removes_only_that_key() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + for i in 0..200 { + t.insert(i, i as u64 + 1).unwrap(); + } + + assert!(t.delete(50).unwrap(), "删一个存在的键应该返回 true"); + assert_eq!(t.get(50).unwrap(), None); + assert_eq!(t.get(49).unwrap(), Some(50)); + assert_eq!(t.get(51).unwrap(), Some(52)); + + // 再删一次:它已经不在了 + assert!(!t.delete(50).unwrap()); + + // 遍历里也不能再有它 + let all = t.iter_all().unwrap(); + assert_eq!(all.len(), 199); + assert!(all.iter().all(|(k, _)| *k != 50)); +} + +/// 删过之后树还能继续插(惰性删除会留下稀疏节点,但不能影响正确性) +#[test] +fn insert_after_delete_still_works() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + for i in 0..300 { + t.insert(i, i as u64).unwrap(); + } + // 把偶数键全删掉 + for i in (0..300).step_by(2) { + assert!(t.delete(i).unwrap()); + } + // 再插一批新的 + for i in 1000..1100 { + t.insert(i, i as u64).unwrap(); + } + for i in 0..300 { + let want = if i % 2 == 0 { None } else { Some(i as u64) }; + assert_eq!(t.get(i).unwrap(), want, "key {}", i); + } + for i in 1000..1100 { + assert_eq!(t.get(i).unwrap(), Some(i as u64)); + } +} diff --git a/chusql-storage/tests/heap_test.rs b/chusql-storage/tests/heap_test.rs index 624113b..6c8eb3c 100644 --- a/chusql-storage/tests/heap_test.rs +++ b/chusql-storage/tests/heap_test.rs @@ -21,7 +21,7 @@ fn insert_and_scan_one_row() { let path = dir.path().join("t.db"); let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); - t.insert(&row(&[("id", json!(1)), ("name", json!("Alice"))])) + t.insert_row(&row(&[("id", json!(1)), ("name", json!("Alice"))])) .unwrap(); let rows = t.scan().unwrap(); @@ -38,7 +38,7 @@ fn insert_many_rows_cross_pages() { let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); for i in 0..500 { - t.insert(&row(&[("id", json!(i))])).unwrap(); + t.insert_row(&row(&[("id", json!(i))])).unwrap(); } let rows = t.scan().unwrap(); @@ -56,7 +56,7 @@ fn large_row_still_fits() { let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); let big = "x".repeat(2000); - t.insert(&row(&[("data", json!(big.clone()))])).unwrap(); + t.insert_row(&row(&[("data", json!(big.clone()))])).unwrap(); let rows = t.scan().unwrap(); assert_eq!(rows.len(), 1); @@ -71,7 +71,7 @@ fn row_too_large_errors() { let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); let huge = "x".repeat(10_000); - let err = t.insert(&row(&[("data", json!(huge))])); + let err = t.insert_row(&row(&[("data", json!(huge))])); assert!(err.is_err()); } @@ -83,8 +83,8 @@ fn reopen_and_scan() { { let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); - t.insert(&row(&[("id", json!(1))])).unwrap(); - t.insert(&row(&[("id", json!(2))])).unwrap(); + t.insert_row(&row(&[("id", json!(1))])).unwrap(); + t.insert_row(&row(&[("id", json!(2))])).unwrap(); } let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); @@ -102,12 +102,12 @@ fn custom_page_size() { let mut t = HeapTable::open(&path, 512, DEFAULT_POOL_SIZE).unwrap(); for i in 0..50 { - t.insert(&row(&[("id", json!(i))])).unwrap(); + t.insert_row(&row(&[("id", json!(i))])).unwrap(); } assert_eq!(t.scan().unwrap().len(), 50); let big = "x".repeat(600); - assert!(t.insert(&row(&[("data", json!(big))])).is_err()); + assert!(t.insert_row(&row(&[("data", json!(big))])).is_err()); } /// 整表改写后索引重建 @@ -123,9 +123,9 @@ fn replace_all_rebuilds_index() { ) .unwrap(); - t.insert_keyed(&row(&[("id", json!(1)), ("name", json!("Alice"))]), 1) + t.insert_row(&row(&[("id", json!(1)), ("name", json!("Alice"))])) .unwrap(); - t.insert_keyed(&row(&[("id", json!(2)), ("name", json!("Bob"))]), 2) + t.insert_row(&row(&[("id", json!(2)), ("name", json!("Bob"))])) .unwrap(); assert!(t.get_by_key(1).unwrap().is_some()); @@ -141,3 +141,150 @@ fn replace_all_rebuilds_index() { assert_eq!(got["name"], json!("Zoe")); assert_eq!(t.scan().unwrap().len(), 2); } + +/// 行级删除:行没了、索引条目没了、顺序不乱 +#[test] +fn delete_by_keys_removes_rows() { + let dir = tempfile::tempdir().unwrap(); + let mut t = HeapTable::open_indexed( + dir.path().join("t.db"), + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + + for i in 1..=5 { + t.insert_row(&row(&[("id", json!(i)), ("name", json!(format!("u{}", i)))])) + .unwrap(); + } + + let deleted = t.delete_by_keys(&[2, 4, 99]).unwrap(); + assert_eq!(deleted.len(), 2, "只有 2、4 真的删掉了"); + + let ids: Vec = t + .scan() + .unwrap() + .iter() + .map(|r| r["id"].as_i64().unwrap()) + .collect(); + assert_eq!(ids, vec![1, 3, 5], "剩下的行要保持原顺序"); + + assert!(t.get_by_key(2).unwrap().is_none(), "索引条目要跟着删"); + assert!(t.get_by_key(4).unwrap().is_none()); + assert!(t.get_by_key(3).unwrap().is_some()); + + // 删掉的位置可以再插回来 + t.insert_row(&row(&[("id", json!(2)), ("name", json!("u2b"))])) + .unwrap(); + assert_eq!(t.get_by_key(2).unwrap().unwrap()["name"], json!("u2b")); +} + +/// 批量插入 +#[test] +fn insert_rows_batch() { + let dir = tempfile::tempdir().unwrap(); + let mut t = HeapTable::open_indexed( + dir.path().join("t.db"), + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + + let rows: Vec = (1..=20) + .map(|i| row(&[("id", json!(i)), ("name", json!(format!("u{}", i)))])) + .collect(); + t.insert_rows(&rows).unwrap(); + + assert_eq!(t.scan().unwrap().len(), 20); + assert_eq!(t.get_by_key(20).unwrap().unwrap()["name"], json!("u20")); + + // 批里出现重复键要报错(整批之前就查出来,不会插一半) + let dup = vec![ + row(&[("id", json!(100))]), + row(&[("id", json!(100))]), + ]; + assert!(t.insert_rows(&dup).is_err()); +} + +/// 第二列索引:建索引时要求唯一,建完之后插入/删除都跟着维护 +#[test] +fn secondary_index_is_maintained() { + let dir = tempfile::tempdir().unwrap(); + let mut t = HeapTable::open_indexed( + dir.path().join("t.db"), + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + + t.insert_rows(&[ + row(&[("id", json!(1)), ("code", json!(7001))]), + row(&[("id", json!(2)), ("code", json!(7002))]), + ]) + .unwrap(); + + assert!(!t.has_index("code")); + t.build_index( + "code", + &dir.path().join("t.code.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + assert!(t.has_index("code")); + assert_eq!(t.get_by_column_key("code", 7002).unwrap().unwrap()["id"], json!(2)); + + // 建索引之后新插入的行也进索引 + t.insert_row(&row(&[("id", json!(3)), ("code", json!(7003))])) + .unwrap(); + assert_eq!(t.get_by_column_key("code", 7003).unwrap().unwrap()["id"], json!(3)); + + // 删掉一行,索引条目跟着走 + t.delete_by_keys(&[2]).unwrap(); + assert!(t.get_by_column_key("code", 7002).unwrap().is_none()); + + // code 已经有索引,重复的 code 插不进去(索引列是唯一的) + assert!(t.insert_row(&row(&[("id", json!(4)), ("code", json!(7001))])).is_err()); + let mut t2 = HeapTable::open(&dir.path().join("d.db"), DEFAULT_PAGE_SIZE, DEFAULT_POOL_SIZE).unwrap(); + t2.insert_rows(&[ + row(&[("id", json!(1)), ("age", json!(30))]), + row(&[("id", json!(2)), ("age", json!(30))]), + ]) + .unwrap(); + let err = t2.build_index( + "age", + &dir.path().join("d.age.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ); + assert!(err.is_err(), "重复值不给建索引"); + assert!(!t2.has_index("age"), "失败之后不留半成品"); +} + +/// 同一个 id 插两次要报错 +#[test] +fn duplicate_key_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let mut t = HeapTable::open_indexed( + dir.path().join("t.db"), + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + DEFAULT_POOL_SIZE, + ) + .unwrap(); + + t.insert_row(&row(&[("id", json!(1))])).unwrap(); + let r = t.insert_row(&row(&[("id", json!(1))])); + assert!(r.is_err()); + assert_eq!(t.scan().unwrap().len(), 1, "失败的那一行不应该进堆表"); +} + diff --git a/chusql-storage/tests/pipe_test.rs b/chusql-storage/tests/pipe_test.rs index 23283ef..431c276 100644 --- a/chusql-storage/tests/pipe_test.rs +++ b/chusql-storage/tests/pipe_test.rs @@ -152,17 +152,17 @@ fn replace_all_replaces_rows() { assert!(!r.contains("Alice"), "old row should be gone: {}", r); } -/// 带 key 插入后能按下标查 +/// 按 id 查(列名走默认值 `id`,老客户端不用改) #[test] -fn insert_with_key_then_lookup() { +fn insert_then_lookup_by_index() { let pipe = unique_pipe_name(); let (_srv, _data) = start_server(&pipe); let mut c = connect(&pipe).unwrap(); for (id, name) in [(1, "Alice"), (2, "Bob"), (3, "Carol")] { let req = format!( - r#"{{"method":"insert","table":"idx_users","row":{{"id":{},"name":"{}"}},"key":{}}}"#, - id, name, id + r#"{{"method":"insert","table":"idx_users","row":{{"id":{},"name":"{}"}}}}"#, + id, name ); let r = send(&mut c, &req); assert!(r.contains(r#""status":"ok""#), "insert: {}", r); @@ -175,16 +175,170 @@ fn insert_with_key_then_lookup() { assert!(r.contains(r#""rows":[]"#), "miss: {}", r); } -/// 不带 key 不建索引 +/// 插入时索引由表自己维护:没给 key,id 也会进索引 #[test] -fn insert_without_key_does_not_index() { +fn insert_indexes_the_id_column_automatically() { let pipe = unique_pipe_name(); let (_srv, _data) = start_server(&pipe); let mut c = connect(&pipe).unwrap(); - send(&mut c, r#"{"method":"insert","table":"nokey","row":{"id":1}}"#); - let r = send(&mut c, r#"{"method":"lookup_by_index","table":"nokey","key":1}"#); - assert!(r.contains(r#""rows":[]"#), "got: {}", r); + send(&mut c, r#"{"method":"insert","table":"auto_idx","row":{"id":7,"name":"Zoe"}}"#); + let r = send( + &mut c, + r#"{"method":"lookup_by_index","table":"auto_idx","column":"id","key":7}"#, + ); + assert!(r.contains("Zoe"), "got: {}", r); +} + +/// 同一个 id 插两次要报错(索引列是唯一的) +#[test] +fn duplicate_id_is_rejected() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + let req = r#"{"method":"insert","table":"dup_id","row":{"id":1,"name":"A"}}"#; + assert!(send(&mut c, req).contains(r#""status":"ok""#)); + + let r = send(&mut c, r#"{"method":"insert","table":"dup_id","row":{"id":1,"name":"B"}}"#); + assert!(r.contains(r#""status":"error""#), "got: {}", r); + assert!(r.contains("duplicate key"), "got: {}", r); +} + +/// 批量插入一次全进 +#[test] +fn insert_batch_writes_all_rows() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + let r = send( + &mut c, + r#"{"method":"insert_batch","table":"batch_t","rows":[{"id":1,"name":"A"},{"id":2,"name":"B"},{"id":3,"name":"C"}]}"#, + ); + assert!(r.contains(r#""status":"ok""#), "batch: {}", r); + + let r = send(&mut c, r#"{"method":"scan","table":"batch_t"}"#); + assert!(r.contains("A") && r.contains("B") && r.contains("C"), "scan: {}", r); + + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"batch_t","key":2}"#); + assert!(r.contains("B"), "index after batch: {}", r); +} + +/// 行级删除:行没了、索引条目也没了、别人不受影响 +#[test] +fn delete_keys_removes_row_and_index_entry() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + for (id, name) in [(1, "Alice"), (2, "Bob"), (3, "Carol")] { + let req = format!( + r#"{{"method":"insert","table":"del_t","row":{{"id":{},"name":"{}"}}}}"#, + id, name + ); + send(&mut c, &req); + } + + let r = send(&mut c, r#"{"method":"delete_keys","table":"del_t","keys":[2]}"#); + assert!(r.contains(r#""status":"ok""#), "delete: {}", r); + + let r = send(&mut c, r#"{"method":"scan","table":"del_t"}"#); + assert!(!r.contains("Bob"), "row should be gone: {}", r); + assert!(r.contains("Alice") && r.contains("Carol"), "others stay: {}", r); + + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"del_t","key":2}"#); + assert!(r.contains(r#""rows":[]"#), "index entry should be gone: {}", r); + + let r = send(&mut c, r#"{"method":"insert","table":"del_t","row":{"id":2,"name":"Bob2"}}"#); + assert!(r.contains(r#""status":"ok""#), "id 2 应该能再插进来: {}", r); +} + +/// 给第二列建索引后能按那一列查 +#[test] +fn create_index_on_secondary_column() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + for (id, code) in [(1, 5001), (2, 5002), (3, 5003)] { + let req = format!( + r#"{{"method":"insert","table":"sec_t","row":{{"id":{},"code":{}}}}}"#, + id, code + ); + send(&mut c, &req); + } + + // 没建之前:明确回 no_index,而不是回空结果 + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"sec_t","column":"code","key":5002}"#); + assert!(r.contains(r#""status":"no_index""#), "got: {}", r); + + let r = send(&mut c, r#"{"method":"create_index","table":"sec_t","column":"code"}"#); + assert!(r.contains(r#""status":"ok""#), "create_index: {}", r); + + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"sec_t","column":"code","key":5002}"#); + assert!(r.contains(r#""id":2"#), "lookup by code: {}", r); + + // 建完索引之后新插入的行也要进索引 + send(&mut c, r#"{"method":"insert","table":"sec_t","row":{"id":4,"code":5004}}"#); + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"sec_t","column":"code","key":5004}"#); + assert!(r.contains(r#""id":4"#), "lookup new row: {}", r); + + // schema 里能看到这条索引 + let r = send(&mut c, r#"{"method":"describe_table","table":"sec_t"}"#); + assert!(r.contains(r#""column":"code""#), "describe: {}", r); + + // 删掉索引后回到 no_index + let r = send(&mut c, r#"{"method":"drop_index","table":"sec_t","column":"code"}"#); + assert!(r.contains(r#""status":"ok""#), "drop_index: {}", r); + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"sec_t","column":"code","key":5002}"#); + assert!(r.contains(r#""status":"no_index""#), "after drop: {}", r); +} + +/// 有重复值的列不给建索引(不然"走索引"和"全表扫"会给出不同答案) +#[test] +fn create_index_rejects_duplicate_values() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + send(&mut c, r#"{"method":"insert","table":"dup_t","row":{"id":1,"age":30}}"#); + send(&mut c, r#"{"method":"insert","table":"dup_t","row":{"id":2,"age":30}}"#); + + let r = send(&mut c, r#"{"method":"create_index","table":"dup_t","column":"age"}"#); + assert!(r.contains(r#""status":"error""#), "got: {}", r); + assert!(r.contains("duplicate value"), "got: {}", r); + + // 失败之后不留半成品:查这一列仍然是 no_index + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"dup_t","column":"age","key":30}"#); + assert!(r.contains(r#""status":"no_index""#), "got: {}", r); +} + +/// 数据字典里的统计:不同值个数 +#[test] +fn stats_report_distinct_values() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + for (id, age) in [(1, 30), (2, 30), (3, 41)] { + let req = format!( + r#"{{"method":"insert","table":"stat_t","row":{{"id":{},"age":{}}}}}"#, + id, age + ); + send(&mut c, &req); + } + + let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#); + assert!(r.contains(r#""row_count":3"#), "row_count: {}", r); + assert!(r.contains(r#""name":"age","distinct":2"#), "age 有 2 个不同值: {}", r); + assert!(r.contains(r#""name":"id","distinct":3"#), "id 有 3 个不同值: {}", r); + + // 删掉一行,统计跟着掉 + send(&mut c, r#"{"method":"delete_keys","table":"stat_t","keys":[1]}"#); + let r = send(&mut c, r#"{"method":"describe_table","table":"stat_t"}"#); + assert!(r.contains(r#""row_count":2"#), "after delete: {}", r); + assert!(r.contains(r#""name":"age","distinct":1"#), "after delete: {}", r); } /// 重开服务后索引仍在 diff --git a/chusql-storage/tests/wal_test.rs b/chusql-storage/tests/wal_test.rs index d0feeb2..eccdb22 100644 --- a/chusql-storage/tests/wal_test.rs +++ b/chusql-storage/tests/wal_test.rs @@ -22,7 +22,6 @@ fn append_and_read_insert() { let op = WalOp::Insert { table: "users".into(), row: row(&[("id", json!(1)), ("name", json!("Alice"))]), - key: Some(1), }; wal.append(&op).unwrap(); @@ -77,7 +76,6 @@ fn clear_truncates() { wal.append(&WalOp::Insert { table: "t".into(), row: row(&[("id", json!(1))]), - key: None, }) .unwrap(); assert!(wal.read().unwrap().is_some()); @@ -97,7 +95,6 @@ fn truncated_tail_is_ignored() { wal.append(&WalOp::Insert { table: "users".into(), row: row(&[("id", json!(1)), ("name", json!("Alice"))]), - key: Some(1), }) .unwrap(); @@ -116,9 +113,9 @@ fn overwrite_on_second_append_is_visible() { wal.append(&WalOp::Insert { table: "t".into(), row: row(&[("id", json!(1))]), - key: None, }) .unwrap(); let first = wal.read().unwrap().unwrap(); assert!(matches!(first, WalOp::Insert { .. })); } +