From 0791fe04469da03c6b3e9f8fe2583ae67c6044ff Mon Sep 17 00:00:00 2001 From: LIm1tZz7 <36764786+TheBadRoger@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:44:37 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E6=9F=A5=E8=AF=A2=E4=BC=98?= =?UTF-8?q?=E5=8C=96=EF=BC=9A=E8=B0=93=E8=AF=8D=E4=B8=8B=E6=8E=A8=E3=80=81?= =?UTF-8?q?=E6=8A=95=E5=BD=B1=E4=B8=8B=E6=8E=A8=E5=92=8C=E5=B8=B8=E9=87=8F?= =?UTF-8?q?=E6=8A=98=E5=8F=A0=EF=BC=8C=E9=85=8D=E7=BD=AE=E5=A5=BD=E4=BA=86?= =?UTF-8?q?benchmark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 31 +++ benchmark/README.md | 93 +++++++ benchmark/chusql-engine/chusql-bench.cabal | 14 ++ benchmark/chusql-engine/src/Main.hs | 118 +++++++++ benchmark/chusql-engine/stack.yaml | 5 + benchmark/chusql-engine/stack.yaml.lock | 12 + benchmark/chusql-storage/README.md | 36 +++ chusql-engine/app/Main.hs | 67 ++++- chusql-engine/chusql-engine.cabal | 1 + chusql-engine/src/ChuSQL/Algebra/Optimize.hs | 162 ++++++++++++ chusql-engine/src/ChuSQL/SQLSyntax/AST.hs | 1 + .../src/ChuSQL/SQLSyntax/Executor.hs | 3 +- chusql-engine/src/ChuSQL/SQLSyntax/Expr.hs | 1 + chusql-engine/test/Spec.hs | 234 ++++++++++++++++++ 14 files changed, 771 insertions(+), 7 deletions(-) create mode 100644 benchmark/README.md create mode 100644 benchmark/chusql-engine/chusql-bench.cabal create mode 100644 benchmark/chusql-engine/src/Main.hs create mode 100644 benchmark/chusql-engine/stack.yaml create mode 100644 benchmark/chusql-engine/stack.yaml.lock create mode 100644 benchmark/chusql-storage/README.md create mode 100644 chusql-engine/src/ChuSQL/Algebra/Optimize.hs diff --git a/README.md b/README.md index ea830f3..4dd907b 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,34 @@ > Rust和Haskell部分的通信通过管道实现，后期跨平台到Linux后使用Unix Domain Socket > 如果找到了较好的Rust+Haskell集成方法也会及时替换 + +# 代码规模 + +| 语言 / 类别 | 文件数 | 行数 | 说明 | +| --------------- | -----: | -------: | --------------------------------------------------------------- | +| Haskell（引擎） | 11 | 902 | `chusql-engine/src/` 10 个模块 + `app/Main.hs` | +| Haskell（测试） | 1 | 1058 | `test/Spec.hs`，124 条用例 | +| Haskell（基准） | 1 | 118 | `benchmark/chusql-engine/src/Main.hs` | +| Rust | 0 | 0 | `chusql-storage/` 尚未开工 | +| 前端 | 0 | 0 | `chusql-web/` 尚未开工 | +| Markdown 文档 | 6 | 1456 | `README.md` + `docs/` 3 篇 + `benchmark/` 2 篇 | +| 构建 / 配置 | 9 | 231 | cabal、`stack.yaml(.lock)`、`tasks.json`、`.gitignore`、LICENSE | +| 项目计划 | 1 | 368 | `chusql-engine/projectplan.txt` | +| **合计** | **29** | **4133** | 另有 3 个 `.gitkeep` 占位文件未计入 | + +> 最近更新：2026-09-19 + +# 性能基准 + +**测试环境**：Windows / GHC 9.10.3 / `-O2` +**数据规模**：`users`、`orders` 各 500 行，两表配对共 25 万对 +**计时方式**：CPU 时间，自适应批量（累计超过 0.3 秒后折算成单次耗时） + +| 查询 | 结果行数 | 未优化 | 优化后 | 加速 | +| ------------------------------------------ | -------: | -------: | ----------: | --------: | +| `JOIN` + `WHERE u.age > 90` | 45 | 125.0 ms | **7.1 ms** | **17.7x** | +| `JOIN`（无 WHERE，只有投影） | 500 | 132.8 ms | 105.5 ms | 1.3x | +| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 171.9 ms | **10.3 ms** | **16.8x** | +| 单表 `WHERE age > 90`（对照组） | 45 | 0.111 ms | 0.153 ms | 0.7x | + +> 最近更新：2026-09-19 diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..c8fd04e --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,93 @@ +# ChuSQL 基准测试 + +这个目录放**性能基准测试**，和 `chusql-engine`、`chusql-storage` 的功能代码分开， +因为它回答的问题不一样：功能代码回答"对不对"，基准测试回答"快不快、快多少"。 + +``` +benchmark/ +├── README.md 本文件：怎么跑、测什么、结果怎么解读 +├── chusql-engine/ Haskell 查询层：优化器开/关的对比 +│ ├── stack.yaml 把 chusql-engine 作为本地包引入 +│ ├── chusql-bench.cabal +│ └── src/Main.hs +└── chusql-storage/ Rust 存储层：预留（见该目录下的 README） +``` + +--- + +## 怎么跑 + +```bash +cd benchmark/chusql-engine +stack run # 默认 users=500 行、orders=500 行 +stack run -- 1000 1000 0.5 # 也可以自己指定：users 行数、orders 行数、每批目标秒数 +``` + +第一次跑要编译，之后很快。整个默认跑完约 5 秒。 + +--- + +## 测什么 + +四个用例，**每个都跑两遍**：一遍走"翻译出来的原始工序单"，一遍走"优化后的工序单"。 +两次的结果行数会被打印出来互相对照，**不一致会当场标出来**。 + +| 用例 | 想看什么 | +| ---- | -------- | +| `JOIN` + `WHERE u.age > 90` | 谓词下推的主场：条件能筛掉约 90% 的行 | +| `JOIN`（无 WHERE） | 只剩投影下推在起作用，收益应该温和 | +| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT` | 规则叠加时的综合效果 | +| 单表 `WHERE age > 90` | 对照组：没有下推空间，优化器只剩固定开销 | + +--- + +## 计时怎么做（两个坑都踩过了） + +**坑一：Haskell 是惰性的，不强制求值就等于没测。** +如果只写 `let r = evalRelOp db plan`，`r` 根本没被算出来，测到的是"定义一个变量"的时间。 +所以每次必须把结果**算到底**（本项目里是逐行把列数累加起来当校验和）。 + +**坑二：Windows 上 CPU 时钟的精度只有约 15.6 毫秒。** +优化后的查询只要几毫秒，直接测会得到 `0.0 ms` 这种没意义的数字。 + +所以计时用的是**自适应批量**：把同一个用例连续跑 `k` 次测一整批，`k` 不断翻倍， +直到一批累计耗时超过目标值（默认 0.3 秒），再折算成单次耗时。 +这样时钟精度的误差会被 `k` 摊薄，几毫秒的用例也能测准。 +程序会打印实际用了多大的批次（例如"优化后 64 次/批"）。 + +跑正式计时之前还会**预热两遍**，避免把首次构造表数据的开销算进去。 + +--- + +## 实测结果 + +**环境**：Windows，GHC 9.10.3，`-O2` +**数据**：`users` 500 行（`id / name / age`，`age` 只有 0-99 共 100 种取值）， +`orders` 500 行（`id / user_id / product`），两表配对共 25 万对。 + +| 查询 | 结果行数 | 未优化 | 优化后 | 加速 | +| ---- | -------- | ------ | ------ | ---- | +| `JOIN` + `WHERE u.age > 90` | 45 | 125.0 ms | **7.1 ms** | **17.7x** | +| `JOIN`（无 WHERE） | 500 | 132.8 ms | 105.5 ms | 1.3x | +| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 171.9 ms | **10.3 ms** | **16.8x** | +| 单表 `WHERE age > 90` | 45 | 0.111 ms | 0.153 ms | 0.7x | + +### 怎么解读 + +- **前两行的差距来自谓词下推。** 条件筛掉了约 90% 的用户，配对次数从 25 万降到约 2.25 万， + 时间直接掉一个数量级。筛得越狠，省得越多。 +- **第二行只有投影下推。** 配对次数一点没变（没有条件可下推），只是每一行变窄了， + 所以只有 1.3 倍 —— 收益温和但确实是正的。 +- **第四行反而略慢。** 单表查询本来就没有下推空间（条件早已贴着扫描）， + 而优化器自己要走一遍规则，有几十微秒的固定开销（0.153 − 0.111 ≈ 0.04 ms）。 + **优化不是零成本的**：查询越大、配对越多，这笔账才越划算。 + +--- + +## 加新用例 + +在 `src/Main.hs` 的 `queries` 列表里加一条 SQL 字符串就行，其余全自动 +（会自动翻译、优化、跑两遍、比结果、算加速比）。 + +想测别的规模，直接命令行传参；想在文档里引用数字，记得把**数据规模一起写上** —— +脱离规模谈"快了几倍"是没有意义的。 diff --git a/benchmark/chusql-engine/chusql-bench.cabal b/benchmark/chusql-engine/chusql-bench.cabal new file mode 100644 index 0000000..d8c6c9b --- /dev/null +++ b/benchmark/chusql-engine/chusql-bench.cabal @@ -0,0 +1,14 @@ +cabal-version: 2.4 + +name: chusql-bench +version: 1.0.0 +synopsis: ChuSQL 查询优化基准测试（对比开/关优化器） +build-type: Simple + +executable chusql-bench + main-is: Main.hs + hs-source-dirs: src + build-depends: base >= 4.14 && < 5 + , chusql-engine + default-language: Haskell2010 + ghc-options: -O2 -Wall diff --git a/benchmark/chusql-engine/src/Main.hs b/benchmark/chusql-engine/src/Main.hs new file mode 100644 index 0000000..646b0b5 --- /dev/null +++ b/benchmark/chusql-engine/src/Main.hs @@ -0,0 +1,118 @@ +module Main where + +import ChuSQL.Algebra.Eval (evalRelOp) +import ChuSQL.Algebra.Optimize (optimize) +import ChuSQL.Algebra.Planner (translate) +import ChuSQL.Model +import ChuSQL.SQLSyntax.Parser (parseQuery) +import Control.Exception (evaluate) +import System.CPUTime (getCPUTime) +import System.Environment (getArgs) +import Text.Printf (printf) + +mkUsers :: Int -> Table +mkUsers n = + Table + { tableName = "users" + , tableCols = ["id", "name", "age"] + , tableRows = + [ [("id", VInt i), ("name", VStr ("user" ++ show i)), ("age", VInt (i `mod` 100))] + | i <- [1 .. n] + ] + } + +mkOrders :: Int -> Int -> Table +mkOrders n m = + Table + { tableName = "orders" + , tableCols = ["id", "user_id", "product"] + , tableRows = + [ [("id", VInt j), ("user_id", VInt (1 + (j - 1) `mod` n)), ("product", VStr ("item" ++ show (j `mod` 50)))] + | j <- [1 .. m] + ] + } + +checksum :: Either String [Row] -> Int +checksum (Left _) = -1 +checksum (Right rows) = go 0 rows + where + go acc [] = acc + go acc (r : rs) = go (acc + length r) rs + +rowCount :: Either String [Row] -> Int +rowCount (Left _) = -1 +rowCount (Right rows) = length rows + +runOnce :: Database -> String -> Bool -> IO Int +runOnce db sql useOpt = do + let result = case parseQuery sql >>= translate db of + Left e -> Left e + Right plan -> evalRelOp db (if useOpt then optimize db plan else plan) + forced <- evaluate (checksum result) + return forced + +resultRows :: Database -> String -> Bool -> IO Int +resultRows db sql useOpt = do + let result = case parseQuery sql >>= translate db of + Left e -> Left e + Right plan -> evalRelOp db (if useOpt then optimize db plan else plan) + _ <- evaluate (checksum result) + return (rowCount result) + +timed :: IO a -> IO Double +timed act = do + t0 <- getCPUTime + _ <- act + t1 <- getCPUTime + return (fromIntegral (t1 - t0) / 1e12) + +benchOp :: Double -> IO Int -> IO (Double, Int) +benchOp target act = loop 1 + where + loop k = do + t <- timed (mapM_ (const act) [1 .. k]) + if t < target then loop (k * 2) else return (t / fromIntegral k, k) + +report :: Database -> Double -> String -> IO () +report db target sql = do + nUn <- resultRows db sql False + nOp <- resultRows db sql True + (tUn, kUn) <- benchOp target (runOnce db sql False) + (tOp, kOp) <- benchOp target (runOnce db sql True) + let speedup = if tOp <= 0 then 0 else tUn / tOp + mismatch = if nUn == nOp then "" else " <<< 结果不一致！" + printf "%s\n" sql + printf + " 结果行数 %-6d | 未优化 %9.3f ms | 优化后 %9.3f ms | 加速 %6.1fx%s\n" + nOp + (tUn * 1000) + (tOp * 1000) + speedup + mismatch + printf " （实测批次：未优化 %d 次/批，优化后 %d 次/批）\n\n" kUn kOp + +queries :: [String] +queries = + [ "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 90" + , "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id" + , "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 90 ORDER BY o.product LIMIT 10" + , "SELECT name FROM users WHERE age > 90" + ] + +warmupSql :: String +warmupSql = "SELECT name FROM users" + +main :: IO () +main = do + args <- getArgs + let pick i d = if length args > i then read (args !! i) else d + n = pick 0 500 + m = pick 1 500 + target = pick 2 (0.3 :: Double) + db = [("users", mkUsers n), ("orders", mkOrders n m)] + printf "数据规模：users = %d 行，orders = %d 行\n" n m + printf "计时方式：CPU 时间；每个用例自动加倍批量次数，直到一批累计耗时超过 %.2f 秒，再折算成单次耗时\n" target + printf "（先跑一遍预热，避免把首次构造数据的开销算进去）\n\n" + _ <- runOnce db warmupSql False + _ <- runOnce db warmupSql True + mapM_ (report db target) queries diff --git a/benchmark/chusql-engine/stack.yaml b/benchmark/chusql-engine/stack.yaml new file mode 100644 index 0000000..eacce59 --- /dev/null +++ b/benchmark/chusql-engine/stack.yaml @@ -0,0 +1,5 @@ +resolver: lts-24.59 + +packages: + - . + - ../../chusql-engine diff --git a/benchmark/chusql-engine/stack.yaml.lock b/benchmark/chusql-engine/stack.yaml.lock new file mode 100644 index 0000000..e3f9041 --- /dev/null +++ b/benchmark/chusql-engine/stack.yaml.lock @@ -0,0 +1,12 @@ +# This file was autogenerated by Stack. +# You should not edit this file by hand. +# For more information, please see the documentation at: +# https://docs.haskellstack.org/en/stable/topics/lock_files + +packages: [] +snapshots: +- completed: + sha256: 0f728e30e843d7f00460ad56858b995a3b227fca683659c7df9f2fc3988a3ce1 + size: 732465 + url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/24/59.yaml + original: lts-24.59 diff --git a/benchmark/chusql-storage/README.md b/benchmark/chusql-storage/README.md new file mode 100644 index 0000000..9e539cd --- /dev/null +++ b/benchmark/chusql-storage/README.md @@ -0,0 +1,36 @@ +# chusql-storage 基准测试（预留） + +这个目录预留给 **Rust 存储层** 的基准测试代码，目前 `chusql-storage/` 还没有开工， +所以这里先占位。 + +## 计划放什么 + +`chusql-storage` 做的是磁盘那一层的事情，性能指标和查询层完全不同 —— +查询层关心"少算几行"，存储层关心"少读几次盘"。所以基准要分开测： + +| 待测模块 | 关心的指标 | +| -------- | ---------- | +| 页管理 | 单页读写延迟、顺序读 vs 随机读吞吐 | +| B+ 树 | 插入/查找/范围扫描的耗时随数据量增长的趋势 | +| 缓冲池 | 命中率、缺页次数、替换策略（LRU 等）的对比 | +| WAL | 追加写吞吐、日志落盘延迟、恢复耗时 | +| 崩溃恢复 | 从日志重放的时间、恢复后数据一致性 | + +## 用什么 + +Rust 侧的基准测试一般用 [criterion](https://github.com/bheisler/criterion.rs)： + +``` +benchmark/chusql-storage/ +├── Cargo.toml +└── benches/ + ├── page.rs + ├── btree.rs + ├── buffer_pool.rs + └── wal.rs +``` + +跑法：`cargo bench` + +另外还要和 Haskell 层对齐口径：查询层的基准是"整条查询的耗时"， +存储层的基准是"单次 IO / 单次查找的耗时"，两边不要混在一起比。 diff --git a/chusql-engine/app/Main.hs b/chusql-engine/app/Main.hs index 03a6fdc..613428c 100644 --- a/chusql-engine/app/Main.hs +++ b/chusql-engine/app/Main.hs @@ -1,24 +1,39 @@ module Main where -import ChuSQL.SQLSyntax.Executor +import ChuSQL.Algebra.Op (RelOp (..)) +import ChuSQL.Algebra.Optimize (optimize) +import ChuSQL.Algebra.Planner (translate) import ChuSQL.Model import ChuSQL.SQLSyntax.AST +import ChuSQL.SQLSyntax.Executor import ChuSQL.SQLSyntax.Parser users :: Table users = Table { tableName = "users" - , tableCols = ["name", "age"] + , tableCols = ["id", "name", "age"] , tableRows = - [ [("name", VStr "Alice"), ("age", VInt 25)] - , [("name", VStr "Bob"), ("age", VInt 17)] - , [("name", VStr "Carol"), ("age", VInt 30)] + [ [("id", VInt 1), ("name", VStr "Alice"), ("age", VInt 25)] + , [("id", VInt 2), ("name", VStr "Bob"), ("age", VInt 17)] + , [("id", VInt 3), ("name", VStr "Carol"), ("age", VInt 30)] + ] + } + +orders :: Table +orders = + Table + { tableName = "orders" + , tableCols = ["id", "user_id", "product"] + , tableRows = + [ [("id", VInt 1), ("user_id", VInt 1), ("product", VStr "Book")] + , [("id", VInt 2), ("user_id", VInt 2), ("product", VStr "Pen")] + , [("id", VInt 3), ("user_id", VInt 1), ("product", VStr "Cup")] ] } db :: Database -db = [("users", users)] +db = [("users", users), ("orders", orders)] -- runQuery 返回 (更新后的数据库, 结果行)；演示只关心结果行 rowsOf :: Either String (Database, [Row]) -> Either String [Row] @@ -26,6 +41,7 @@ rowsOf = fmap snd main :: IO () main = do + putStrLn "==================== Parser Test =====================" let q1 = makeSelect ["name"] "users" (Just (Gt (Col "age") (LitInt 18))) print (rowsOf (runQuery db q1)) @@ -50,3 +66,42 @@ main = do -- 解析 SQL 文本，再交给执行引擎 let parsed = parseQuery "SELECT name FROM users WHERE age > 18 AND name = 'Alice'" print (rowsOf (parsed >>= runQuery db)) + + putStrLn "\n==================== Optimizer =====================" + let planSql = + "SELECT u.name, o.product FROM users u \ + \JOIN orders o ON u.id = o.user_id WHERE u.age > 18" + case parseQuery planSql of + Left err -> putStrLn err + Right query -> case translate db query of + Left err -> putStrLn err + Right plan -> do + putStrLn "--- plan before optimization ---" + putStrLn (dump 0 plan) + putStrLn "--- plan after optimization ---" + putStrLn (dump 0 (optimize db plan)) + putStrLn "--- rows ---" + print (rowsOf (runQuery db query)) + +dump :: Int -> RelOp -> String +dump ind op = case op of + Scan a t -> + pad ++ "Scan " ++ show a ++ " " ++ show t + Filter e x -> + pad ++ "Filter " ++ show e ++ "\n" ++ dump (ind + 1) x + Project c x -> + pad ++ "Project " ++ show c ++ "\n" ++ dump (ind + 1) x + Sort s x -> + pad ++ "Sort " ++ show s ++ "\n" ++ dump (ind + 1) x + Limit n x -> + pad ++ "Limit " ++ show n ++ "\n" ++ dump (ind + 1) x + Join l r c -> + pad + ++ "Join on " + ++ show c + ++ "\n" + ++ dump (ind + 1) l + ++ "\n" + ++ dump (ind + 1) r + where + pad = replicate (ind * 2) ' ' diff --git a/chusql-engine/chusql-engine.cabal b/chusql-engine/chusql-engine.cabal index d6dee4d..2ce1c25 100644 --- a/chusql-engine/chusql-engine.cabal +++ b/chusql-engine/chusql-engine.cabal @@ -11,6 +11,7 @@ library hs-source-dirs: src exposed-modules: ChuSQL.Algebra.Eval ChuSQL.Algebra.Op + ChuSQL.Algebra.Optimize ChuSQL.Algebra.Planner ChuSQL.Model ChuSQL.SQLSyntax.AST diff --git a/chusql-engine/src/ChuSQL/Algebra/Optimize.hs b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs new file mode 100644 index 0000000..765c13c --- /dev/null +++ b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs @@ -0,0 +1,162 @@ +module ChuSQL.Algebra.Optimize where + +import ChuSQL.Algebra.Op (RelOp (..)) +import ChuSQL.Model +import ChuSQL.SQLSyntax.AST (Expr (..)) +import ChuSQL.SQLSyntax.Expr (evalExpr) + +-- * 谓词下推 + +-- | 收集表达式里用到的列名；谓词下推判断条件归属、常量折叠判断能否求值，都要用它 +colsInExpr :: Expr -> [String] +colsInExpr (Col c) = [c] +colsInExpr (Gt a b) = colsInExpr a ++ colsInExpr b +colsInExpr (Lt a b) = colsInExpr a ++ colsInExpr b +colsInExpr (Eq a b) = colsInExpr a ++ colsInExpr b +colsInExpr (And a b) = colsInExpr a ++ colsInExpr b +colsInExpr (Or a b) = colsInExpr a ++ colsInExpr b +colsInExpr _ = [] + +-- | 一个算子会产出哪些列；返回 ["*"] 表示"全部列，不用挑" +relOpCols :: Database -> RelOp -> [String] +relOpCols db (Filter _ x) = relOpCols db x +relOpCols db (Sort _ x) = relOpCols db x +relOpCols db (Limit _ x) = relOpCols db x +relOpCols _ (Project ["*"] _) = ["*"] +relOpCols _ (Project cols _) = cols +relOpCols db (Join l r _) = relOpCols db l ++ relOpCols db r +relOpCols db (Scan mAlias tbl) = + case lookup tbl db of + Nothing -> [] + Just t -> map (prefix ++) (tableCols t) + where + prefix = maybe "" (++ ".") mAlias + +-- | need 里的列是不是都能在 have 里找到；have 为 ["*"] 时一律为真 +isSubsetOf :: [String] -> [String] -> Bool +isSubsetOf _ ["*"] = True +isSubsetOf need have = all (`elem` have) need + +-- | 把一个用 AND 串起来的条件拆成清单：a AND b AND c 变成 [a, b, c] +splitConj :: Expr -> [Expr] +splitConj (And a b) = splitConj a ++ splitConj b +splitConj e = [e] + +-- | 反过来：把条件清单用 AND 重新串成一个；空清单表示"没有条件"，返回 Nothing +combineConj :: [Expr] -> Maybe Expr +combineConj [] = Nothing +combineConj (e : es) = Just (foldl And e es) + +-- | 把条件按"只用到哪一边的列"分成两堆：能下推过去的 / 必须留在原地的 +partitionPred :: [String] -> [Expr] -> ([Expr], [Expr]) +partitionPred have = foldr pick ([], []) + where + pick e (yes, no) + | colsInExpr e `isSubsetOf` have = (e : yes, no) + | otherwise = (yes, e : no) + +-- | 把一组条件依次套到算子上，一个条件一层 Filter +applyFilters :: [Expr] -> RelOp -> RelOp +applyFilters [] op = op +applyFilters (e : es) op = applyFilters es (Filter e op) + +-- | Filter 压在 Join 上时的下推规则：条件按列归属拆到左右两侧，跨两边的才留在 Join 上方 +pushJoin :: Database -> Expr -> RelOp -> RelOp -> Expr -> RelOp +pushJoin db p l r c = + let preds = splitConj p + lCols = relOpCols db l + rCols = relOpCols db r + (lPs, rest1) = partitionPred lCols preds + (rPs, bothPs) = partitionPred rCols rest1 + l' = applyFilters lPs l + r' = applyFilters rPs r + joined = Join l' r' c + in case combineConj bothPs of + Nothing -> joined + Just p' -> Filter p' joined + +-- | Filter 压在 Filter / Sort 上时的下推规则：合并上下两层 Filter，或把 Filter 挪到 Sort 下面 +pushOne :: Database -> RelOp -> RelOp +pushOne _ (Filter p (Filter q x)) = Filter (And p q) x +pushOne _ (Filter p (Sort spec x)) = Sort spec (Filter p x) +pushOne _ op = op + +-- * 投影下推 + +-- | 工具函数 +needsAll :: [String] -> Bool +needsAll = elem "*" + +pushProject :: Database -> [String] -> RelOp -> RelOp +pushProject db _ (Project cols x) = case pushProject db cols x of + Project cols' y | cols' == cols -> Project cols y + other -> Project cols other +pushProject db need (Filter p x) = Filter p (pushProject db (need ++ colsInExpr p) x) +pushProject db need (Sort spec x) = Sort spec (pushProject db (need ++ map fst spec) x) +pushProject db need (Limit n x) = Limit n (pushProject db need x) +pushProject db need (Join l r c) + | needsAll need = Join (pushProject db need l) (pushProject db need r) c + | otherwise = + let tot = need ++ colsInExpr c + lCols = relOpCols db l + rCols = relOpCols db r + lNeed = if needsAll lCols then ["*"] else [x | x <- tot, x `elem` lCols] + rNeed = if needsAll rCols then ["*"] else [x | x <- tot, x `elem` rCols] + in Join (pushProject db lNeed l) (pushProject db rNeed r) c +pushProject db need scan@(Scan _ _) + | needsAll need = scan + | all (`elem` need) (relOpCols db scan) = scan + | otherwise = Project need scan + +-- * 常量折叠 + +-- | 折一个表达式：不含列就直接求值替换成字面量，含列或求值失败则原样返回 +foldNode :: Expr -> Expr +foldNode e + | null (colsInExpr e) = case evalExpr e [] of + Right (VInt n) -> LitInt n + Right (VStr s) -> LitStr s + Right (VBool b) -> LitBool b + Left _ -> e + | otherwise = e + +-- | 自底向上折叠整个表达式：先把子表达式折干净，再折自己 +foldConstants :: Expr -> Expr +foldConstants (Gt a b) = foldNode (Gt (foldConstants a) (foldConstants b)) +foldConstants (Lt a b) = foldNode (Lt (foldConstants a) (foldConstants b)) +foldConstants (Eq a b) = foldNode (Eq (foldConstants a) (foldConstants b)) +foldConstants (And a b) = foldNode (And (foldConstants a) (foldConstants b)) +foldConstants (Or a b) = foldNode (Or (foldConstants a) (foldConstants b)) +foldConstants e = e + +-- * 执行优化 + +-- | 单节点重写：把 Filter 分派给上面几条规则，另外负责恒真条件删除与恒等投影消除 +rewriteNode :: Database -> RelOp -> RelOp +rewriteNode db (Filter p (Join l r c)) = pushJoin db p l r c +rewriteNode db (Filter p x) = case foldConstants p of + LitBool True -> x + p' -> pushOne db (Filter p' x) +rewriteNode _ (Project ["*"] x) = x +rewriteNode db op = pushOne db op + +-- | 自底向上遍历整棵算子树，每个节点套一次 rewriteNode +optimizeRelOp :: Database -> RelOp -> RelOp +optimizeRelOp db = rewriteNode db . descend + where + descend (Filter p x) = Filter p (optimizeRelOp db x) + descend (Project cols x) = Project cols (optimizeRelOp db x) + descend (Sort spec x) = Sort spec (optimizeRelOp db x) + descend (Limit n x) = Limit n (optimizeRelOp db x) + descend (Join l r c) = Join (optimizeRelOp db l) (optimizeRelOp db r) c + descend x = x + +-- | 谓词下推 + 常量折叠：反复应用规则，直到算子树的形状不再变化（到不动点为止） +optimizePredicates :: Database -> RelOp -> RelOp +optimizePredicates db op = + let op' = optimizeRelOp db op + in if op' == op then op else optimizePredicates db op' + +-- | 优化器总入口：先把谓词下推与常量折叠跑到不动点，再做一次投影下推 +optimize :: Database -> RelOp -> RelOp +optimize db op = pushProject db ["*"] (optimizePredicates db op) diff --git a/chusql-engine/src/ChuSQL/SQLSyntax/AST.hs b/chusql-engine/src/ChuSQL/SQLSyntax/AST.hs index 4c27eb4..93417ef 100644 --- a/chusql-engine/src/ChuSQL/SQLSyntax/AST.hs +++ b/chusql-engine/src/ChuSQL/SQLSyntax/AST.hs @@ -25,6 +25,7 @@ data Expr = Col String | LitInt Int | LitStr String + | LitBool Bool | Gt Expr Expr | Lt Expr Expr | Eq Expr Expr diff --git a/chusql-engine/src/ChuSQL/SQLSyntax/Executor.hs b/chusql-engine/src/ChuSQL/SQLSyntax/Executor.hs index 2c198eb..8320220 100644 --- a/chusql-engine/src/ChuSQL/SQLSyntax/Executor.hs +++ b/chusql-engine/src/ChuSQL/SQLSyntax/Executor.hs @@ -1,6 +1,7 @@ module ChuSQL.SQLSyntax.Executor where import ChuSQL.Algebra.Eval (evalRelOp) +import ChuSQL.Algebra.Optimize (optimize) import ChuSQL.Algebra.Planner (translate) import ChuSQL.Model import ChuSQL.SQLSyntax.AST @@ -28,7 +29,7 @@ applyUpdates ((col, e) : rest) row = do runQuery :: Database -> Query -> Either String (Database, [Row]) runQuery db q@Select{} = do relOp <- translate db q - rows <- evalRelOp db relOp + rows <- evalRelOp db (optimize db relOp) Right (db, rows) -- 处理插入 diff --git a/chusql-engine/src/ChuSQL/SQLSyntax/Expr.hs b/chusql-engine/src/ChuSQL/SQLSyntax/Expr.hs index 85db75b..e9fc0f7 100644 --- a/chusql-engine/src/ChuSQL/SQLSyntax/Expr.hs +++ b/chusql-engine/src/ChuSQL/SQLSyntax/Expr.hs @@ -10,6 +10,7 @@ evalExpr (Col name) row = maybe (Left ("unknown column: " ++ name)) Right (lookup name row) evalExpr (LitInt n) _ = Right (VInt n) evalExpr (LitStr s) _ = Right (VStr s) +evalExpr (LitBool b) _ = Right (VBool b) evalExpr (Gt a b) row = liftBinOp (intOp (>)) a b row evalExpr (Lt a b) row = liftBinOp (intOp (<)) a b row evalExpr (Eq a b) row = liftBinOp eqOp a b row diff --git a/chusql-engine/test/Spec.hs b/chusql-engine/test/Spec.hs index 9388b55..436bd22 100644 --- a/chusql-engine/test/Spec.hs +++ b/chusql-engine/test/Spec.hs @@ -1,5 +1,9 @@ module Main where +import ChuSQL.Algebra.Eval (evalRelOp) +import ChuSQL.Algebra.Op (RelOp (..)) +import ChuSQL.Algebra.Optimize (optimize, pushProject) +import ChuSQL.Algebra.Planner (translate) import ChuSQL.Model import ChuSQL.SQLSyntax.AST import ChuSQL.SQLSyntax.Executor @@ -813,6 +817,169 @@ main = hspec $ do , [("u.name", VStr "Bob"), ("o.product", VStr "Pen")] ] + describe "ChuSQL.Algebra.Optimize" $ do + it "keeps single-table WHERE results identical" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE age > 18" + + it "keeps JOIN with WHERE results identical" $ do + sameResultAsUnoptimized "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 20" + + it "keeps JOIN with WHERE, ORDER BY and LIMIT results identical" $ do + sameResultAsUnoptimized "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 15 ORDER BY o.product DESC LIMIT 2" + + it "keeps SELECT * results identical" $ do + sameResultAsUnoptimized "SELECT * FROM users" + + it "pushes a single-side predicate into the left side of a JOIN" $ do + case parseQuery "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 20" of + Left err -> expectationFailure err + Right q -> + case translate testDB q of + Left err -> expectationFailure err + Right relOp -> + filterPushedIntoLeft (optimize testDB relOp) `shouldBe` True + + it "removes a redundant SELECT * projection" $ do + case parseQuery "SELECT * FROM users" of + Left err -> expectationFailure err + Right q -> + case translate testDB q of + Left err -> expectationFailure err + Right relOp -> + optimize testDB relOp `shouldBe` Scan Nothing "users" + + it "does not push a filter through LIMIT" $ do + let cond = Gt (Col "age") (LitInt 18) + relOp = Filter cond (Limit 2 (Scan Nothing "users")) + optimize testDB relOp `shouldBe` relOp + + it "is idempotent after reaching a fixed point" $ do + case parseQuery "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 20" of + Left err -> expectationFailure err + Right q -> + case translate testDB q of + Left err -> expectationFailure err + Right relOp -> + optimize testDB (optimize testDB relOp) `shouldBe` optimize testDB relOp + + it "folds a constant-false predicate into LitBool False" $ do + fmap firstFilterCond (optimizedPlan "SELECT name FROM users WHERE 1 > 2") + `shouldBe` Right (Just (LitBool False)) + + it "removes a filter whose predicate folds to true" $ do + fmap anyFilter (optimizedPlan "SELECT name FROM users WHERE 1 = 1") + `shouldBe` Right False + + it "removes a constant-true filter sitting above a join" $ do + fmap anyFilter (optimizedPlan "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE 1 = 1") + `shouldBe` Right False + + it "leaves a predicate that mentions a column untouched" $ do + fmap firstFilterCond (optimizedPlan "SELECT name FROM users WHERE age > 18") + `shouldBe` Right (Just (Gt (Col "age") (LitInt 18))) + + it "folds the constant part of a predicate that cannot fold as a whole" $ do + fmap firstFilterCond (optimizedPlan "SELECT name FROM users WHERE age > 18 AND 1 = 1") + `shouldBe` Right (Just (And (Gt (Col "age") (LitInt 18)) (LitBool True))) + + it "keeps a constant predicate that fails to evaluate" $ do + fmap firstFilterCond (optimizedPlan "SELECT name FROM users WHERE 'abc' > 1") + `shouldBe` Right (Just (Gt (LitStr "abc") (LitInt 1))) + + it "keeps constant-false results identical" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE 1 > 2" + + it "keeps constant-true results identical" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE 1 = 1" + + it "keeps mixed constant and column predicates identical" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE age > 18 AND 1 = 1" + + it "keeps a failing constant predicate identical" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE 'abc' > 1" + + it "returns no rows for a constant-false predicate" $ do + rowsOf (parseQuery "SELECT name FROM users WHERE 1 > 2" >>= runQuery testDB) + `shouldBe` Right [] + + it "returns every row for a constant-true predicate" $ do + rowsOf (parseQuery "SELECT name FROM users WHERE 1 = 1" >>= runQuery testDB) + `shouldBe` Right + [ [("name", VStr "Alice")] + , [("name", VStr "Bob")] + , [("name", VStr "Carol")] + ] + + it "keeps the LIMIT node while pushing projection past it" $ do + let plan = Project ["name"] (Limit 2 (Scan Nothing "users")) + stripProjects (pushProject testDB ["*"] plan) `shouldBe` stripProjects plan + + it "keeps the sort keys while pushing projection past a sort" $ do + let plan = Project ["name"] (Sort [("age", Desc)] (Scan Nothing "users")) + stripProjects (pushProject testDB ["*"] plan) `shouldBe` stripProjects plan + + it "keeps both sides of a join" $ do + let plan = + Project + ["u.name"] + ( Join + (Scan (Just "u") "users") + (Scan (Just "o") "orders") + (Eq (Col "u.id") (Col "o.user_id")) + ) + stripProjects (pushProject testDB ["*"] plan) `shouldBe` stripProjects plan + + it "keeps nested projections in place" $ do + let plan = Project ["name"] (Project ["name", "age"] (Scan Nothing "users")) + stripProjects (pushProject testDB ["*"] plan) `shouldBe` stripProjects plan + + it "keeps the whole spine of a full pipeline" $ do + let plan = + Project + ["name"] + ( Limit 2 + ( Sort + [("age", Desc)] + (Filter (Gt (Col "age") (LitInt 18)) (Scan Nothing "users")) + ) + ) + stripProjects (pushProject testDB ["*"] plan) `shouldBe` stripProjects plan + + it "merges two predicates pushed into the same side into one filter" $ do + fmap leftSideFilters (optimizedPlan "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id WHERE u.age > 20 AND u.name = 'Alice'") + `shouldBe` Right (Just 1) + + it "pushes projection into a single-table scan as one layer" $ do + optimizedPlan "SELECT name FROM users" + `shouldBe` Right (Project ["name"] (Scan Nothing "users")) + + it "does not add a projection when the scan already supplies every column" $ do + optimizedPlan "SELECT id, name, age FROM users" + `shouldBe` Right (Project ["id", "name", "age"] (Scan Nothing "users")) + + it "prunes the columns nobody needs on each side of a join" $ do + fmap joinSideCols (projectedPlan "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id") + `shouldBe` Right (Just ["u.name", "u.id"], Just ["o.user_id"]) + + it "does not prune anything when every column is taken" $ do + fmap joinSideCols (projectedPlan "SELECT * FROM users u JOIN orders o ON u.id = o.user_id") + `shouldBe` Right (Nothing, Nothing) + + it "activates projection pushdown inside optimize" $ do + fmap joinSideCols (optimizedPlan "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id") + `shouldBe` Right (Just ["u.name", "u.id"], Just ["o.user_id"]) + + it "keeps projection pushdown idempotent" $ do + case projectedPlan "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id" of + Left err -> expectationFailure err + Right plan -> pushProject testDB ["*"] plan `shouldBe` plan + + it "keeps a join without WHERE results identical" $ do + sameResultAsUnoptimized "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id" + + it "keeps projection with ORDER BY and LIMIT results identical" $ do + sameResultAsUnoptimized "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id ORDER BY o.product DESC LIMIT 2" + -- ============================================================ -- Helpers -- ============================================================ @@ -822,3 +989,70 @@ isLeft (Right _) = False rowsOf :: Either String (Database, [Row]) -> Either String [Row] rowsOf = fmap snd + +sameResultAsUnoptimized :: String -> Expectation +sameResultAsUnoptimized sql = + case parseQuery sql of + Left err -> expectationFailure err + Right q -> + case translate testDB q of + Left err -> expectationFailure err + Right relOp -> + evalRelOp testDB (optimize testDB relOp) `shouldBe` evalRelOp testDB relOp + +filterPushedIntoLeft :: RelOp -> Bool +filterPushedIntoLeft (Project _ (Join (Filter _ _) _ _)) = True +filterPushedIntoLeft _ = False + +optimizedPlan :: String -> Either String RelOp +optimizedPlan sql = do + q <- parseQuery sql + optimize testDB <$> translate testDB q + +firstFilterCond :: RelOp -> Maybe Expr +firstFilterCond (Filter p _) = Just p +firstFilterCond (Project _ x) = firstFilterCond x +firstFilterCond (Sort _ x) = firstFilterCond x +firstFilterCond (Limit _ x) = firstFilterCond x +firstFilterCond (Join l r _) = case firstFilterCond l of + Just p -> Just p + Nothing -> firstFilterCond r +firstFilterCond (Scan _ _) = Nothing + +anyFilter :: RelOp -> Bool +anyFilter (Filter _ _) = True +anyFilter (Project _ x) = anyFilter x +anyFilter (Sort _ x) = anyFilter x +anyFilter (Limit _ x) = anyFilter x +anyFilter (Join l r _) = anyFilter l || anyFilter r +anyFilter (Scan _ _) = False + +stripProjects :: RelOp -> RelOp +stripProjects (Project _ x) = stripProjects x +stripProjects (Filter p x) = Filter p (stripProjects x) +stripProjects (Sort spec x) = Sort spec (stripProjects x) +stripProjects (Limit n x) = Limit n (stripProjects x) +stripProjects (Join l r c) = Join (stripProjects l) (stripProjects r) c +stripProjects (Scan a t) = Scan a t + +projectedPlan :: String -> Either String RelOp +projectedPlan sql = do + q <- parseQuery sql + plan <- translate testDB q + Right (pushProject testDB ["*"] plan) + +sideCols :: RelOp -> Maybe [String] +sideCols (Project cols _) = Just cols +sideCols _ = Nothing + +joinSideCols :: RelOp -> (Maybe [String], Maybe [String]) +joinSideCols (Project _ (Join l r _)) = (sideCols l, sideCols r) +joinSideCols _ = (Nothing, Nothing) + +countFilters :: RelOp -> Int +countFilters (Filter _ x) = 1 + countFilters x +countFilters _ = 0 + +leftSideFilters :: RelOp -> Maybe Int +leftSideFilters (Project _ (Join l _ _)) = Just (countFilters l) +leftSideFilters _ = Nothing