From e42826535796a4fe89d28d9bff0b74a9f87694df Mon Sep 17 00:00:00 2001 From: LIm1tZz7 <36764786+TheBadRoger@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:37:55 +0800 Subject: [PATCH] =?UTF-8?q?rust=E9=83=A8=E5=88=86=E6=8E=A8=E8=BF=9B?= =?UTF-8?q?=EF=BC=9A=E8=BF=9B=E7=A8=8B=E9=97=B4=E9=80=9A=E4=BF=A1=EF=BC=88?= =?UTF-8?q?Windows=E5=B9=B3=E5=8F=B0=E5=91=BD=E5=90=8D=E7=AE=A1=E7=BA=BF?= =?UTF-8?q?=EF=BC=89=E3=80=81=E5=86=85=E5=AD=98=E9=A1=B5=E8=A1=A8=E3=80=81?= =?UTF-8?q?B+=E6=A0=91=E3=80=81WAL=E6=81=A2=E5=A4=8D=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E3=80=81=E5=8A=A8=E6=80=81=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .zed/tasks.json | 49 +- README.md | 6 +- chusql-engine/chusql-engine.cabal | 10 + chusql-engine/src/ChuSQL/Algebra/Eval.hs | 58 ++- chusql-engine/src/ChuSQL/Algebra/Expr.hs | 4 +- chusql-engine/src/ChuSQL/Algebra/Op.hs | 6 +- chusql-engine/src/ChuSQL/Algebra/Optimize.hs | 23 +- chusql-engine/src/ChuSQL/Algebra/Planner.hs | 5 +- chusql-engine/src/ChuSQL/Algebra/Sort.hs | 10 +- chusql-engine/src/ChuSQL/Engine.hs | 126 +++-- chusql-engine/src/ChuSQL/Model.hs | 8 +- chusql-engine/src/ChuSQL/Semantic.hs | 2 + chusql-engine/src/ChuSQL/Storage.hs | 20 + chusql-engine/src/ChuSQL/Storage/.gitkeep | 0 chusql-engine/src/ChuSQL/Storage/IPC.hs | 331 +++++++++++++ chusql-engine/src/ChuSQL/Storage/Memory.hs | 66 +++ chusql-engine/src/ChuSQL/Syntax/AST.hs | 6 +- chusql-engine/src/ChuSQL/Syntax/Parser.hs | 39 +- chusql-engine/test/Spec.hs | 275 +++++++++-- chusql-storage/.gitkeep | 0 chusql-storage/Cargo.lock | 326 +++++++++++++ chusql-storage/Cargo.toml | 13 + chusql-storage/chusql-storage.toml.example | 40 ++ chusql-storage/src/bin/server.rs | 412 ++++++++++++++++ chusql-storage/src/btree.rs | 470 +++++++++++++++++++ chusql-storage/src/catalog.rs | 85 ++++ chusql-storage/src/config.rs | 281 +++++++++++ chusql-storage/src/heap.rs | 267 +++++++++++ chusql-storage/src/lib.rs | 10 + chusql-storage/src/log.rs | 387 +++++++++++++++ chusql-storage/src/page.rs | 120 +++++ chusql-storage/src/protocol.rs | 53 +++ chusql-storage/src/wal.rs | 179 +++++++ chusql-storage/tests/btree_test.rs | 150 ++++++ chusql-storage/tests/catalog_test.rs | 59 +++ chusql-storage/tests/config_test.rs | 131 ++++++ chusql-storage/tests/heap_test.rs | 143 ++++++ chusql-storage/tests/page_test.rs | 129 +++++ chusql-storage/tests/pipe_test.rs | 284 +++++++++++ chusql-storage/tests/wal_test.rs | 119 +++++ 41 files changed, 4546 insertions(+), 159 deletions(-) create mode 100644 chusql-engine/src/ChuSQL/Storage.hs delete mode 100644 chusql-engine/src/ChuSQL/Storage/.gitkeep create mode 100644 chusql-engine/src/ChuSQL/Storage/IPC.hs create mode 100644 chusql-engine/src/ChuSQL/Storage/Memory.hs delete mode 100644 chusql-storage/.gitkeep create mode 100644 chusql-storage/Cargo.lock create mode 100644 chusql-storage/Cargo.toml create mode 100644 chusql-storage/chusql-storage.toml.example create mode 100644 chusql-storage/src/bin/server.rs create mode 100644 chusql-storage/src/btree.rs create mode 100644 chusql-storage/src/catalog.rs create mode 100644 chusql-storage/src/config.rs create mode 100644 chusql-storage/src/heap.rs create mode 100644 chusql-storage/src/lib.rs create mode 100644 chusql-storage/src/log.rs create mode 100644 chusql-storage/src/page.rs create mode 100644 chusql-storage/src/protocol.rs create mode 100644 chusql-storage/src/wal.rs create mode 100644 chusql-storage/tests/btree_test.rs create mode 100644 chusql-storage/tests/catalog_test.rs create mode 100644 chusql-storage/tests/config_test.rs create mode 100644 chusql-storage/tests/heap_test.rs create mode 100644 chusql-storage/tests/page_test.rs create mode 100644 chusql-storage/tests/pipe_test.rs create mode 100644 chusql-storage/tests/wal_test.rs diff --git a/.gitignore b/.gitignore index 349758d..cfb6f09 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ cabal.project.local~ target/ **/*.rs.bk +# 运行时数据（Rust 服务默认数据目录 CHUSQL_DATA_DIR=data/），不入库 +chusql-storage/data/ + # Cargo.lock 要入库（本项目是要产出可执行程序的 crate，不是纯库） # ------------------------------------------------------------ diff --git a/.zed/tasks.json b/.zed/tasks.json index 3a2c583..b70113d 100644 --- a/.zed/tasks.json +++ b/.zed/tasks.json @@ -1,6 +1,6 @@ [ { - "label": "stack: build", + "label": "haskell: stack build", "command": "stack build", "cwd": "$ZED_WORKTREE_ROOT/chusql-engine", "reveal": "always", @@ -8,15 +8,7 @@ "save": "all", }, { - "label": "stack: run", - "command": "stack run", - "cwd": "$ZED_WORKTREE_ROOT/chusql-engine", - "reveal": "always", - "hide": "never", - "save": "all", - }, - { - "label": "stack: test", + "label": "haskell: stack test", "command": "stack test", "cwd": "$ZED_WORKTREE_ROOT/chusql-engine", "reveal": "always", @@ -24,11 +16,46 @@ "save": "all", }, { - "label": "stack: repl", + "label": "haskell: stack run", + "command": "stack run", + "cwd": "$ZED_WORKTREE_ROOT/chusql-engine", + "reveal": "always", + "hide": "never", + "save": "all", + }, + { + "label": "haskell: stack repl", "command": "stack ghci", "cwd": "$ZED_WORKTREE_ROOT/chusql-engine", "reveal": "always", "hide": "never", "save": "all", }, + { + "label": "rust: cargo check", + "command": "cargo", + "args": ["check", "--all-targets"], + "cwd": "$ZED_WORKTREE_ROOT/chusql-storage", + "reveal": "always", + "hide": "on_success", + "save": "all", + }, + { + "label": "rust: cargo test", + "command": "cargo", + "args": ["test"], + "cwd": "$ZED_WORKTREE_ROOT/chusql-storage", + "reveal": "always", + "hide": "never", + "save": "all", + }, + { + "label": "rust: cargo run server", + "command": "cargo", + "args": ["run", "--bin", "server"], + "cwd": "$ZED_WORKTREE_ROOT/chusql-storage", + "reveal": "always", + "hide": "never", + "save": "all", + }, ] diff --git a/README.md b/README.md index 83eeb63..6df3814 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,9 @@ | 查询 | 结果行数 | 未优化 | 优化后 | 加速 | | ------------------------------------------ | -------: | -------: | ----------: | --------: | -| `JOIN` + `WHERE u.age > 90` | 45 | 242.2 ms | **16.6 ms** | **14.6x** | +| `JOIN` + `WHERE u.age > 90` | 45 | 242.2 ms | **16.6 ms** | **14.6x** | | `JOIN`（无 WHERE，只有投影） | 500 | 210.9 ms | 195.3 ms | 1.1x | -| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 164.1 ms | **14.2 ms** | **11.6x** | +| `JOIN` + `WHERE` + `ORDER BY` + `LIMIT 10` | 10 | 164.1 ms | **14.2 ms** | **11.6x** | | 单表 `WHERE age > 90`（对照组） | 45 | 0.175 ms | 0.290 ms | 0.6x | -> 最近更新：2026-09-22（绝对值受机器状态影响，倍率更稳定；同机同法可复现） +> 最近更新：2026-09-22 diff --git a/chusql-engine/chusql-engine.cabal b/chusql-engine/chusql-engine.cabal index 41069e4..f1f839c 100644 --- a/chusql-engine/chusql-engine.cabal +++ b/chusql-engine/chusql-engine.cabal @@ -18,11 +18,18 @@ library ChuSQL.Engine ChuSQL.Model ChuSQL.Semantic + ChuSQL.Storage + ChuSQL.Storage.IPC + ChuSQL.Storage.Memory ChuSQL.Syntax.AST ChuSQL.Syntax.Parser build-depends: base >= 4.14 && < 5 + , aeson + , bytestring , megaparsec , parser-combinators + , scientific + , text default-language: Haskell2010 ghc-options: -Wall @@ -40,6 +47,9 @@ test-suite chusql-engine-test hs-source-dirs: test build-depends: base , chusql-engine + , directory + , filepath , hspec + , process default-language: Haskell2010 ghc-options: -Wall diff --git a/chusql-engine/src/ChuSQL/Algebra/Eval.hs b/chusql-engine/src/ChuSQL/Algebra/Eval.hs index 0bddc5e..7875913 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Eval.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Eval.hs @@ -1,25 +1,41 @@ -module ChuSQL.Algebra.Eval (evalRelOp) where +module ChuSQL.Algebra.Eval (evalRelOp, evalRelOpM) where import ChuSQL.Algebra.Expr (evalCondForRow) import ChuSQL.Algebra.Op import ChuSQL.Algebra.Sort (sortRows) import ChuSQL.Model +import ChuSQL.Storage (MonadStorage (..)) +import ChuSQL.Syntax.AST (Expr (..)) import Control.Monad (filterM) --- 投影 +-- 执行：按关系算子树真正算出结果行（Scan 阶段会给列名加上别名前缀）。 + +-- 投影：只保留清单里的列；哨兵 "*" 表示全部列。 project :: [String] -> Row -> Row project ["*"] row = row project cs row = [(c, v) | (c, v) <- row, c `elem` cs] --- 执行关系代数运算 +-- 给一行里的每个列名加上别名前缀（如 "u."）。 addPrefix :: String -> Row -> Row addPrefix prefix = map (\(k, v) -> (prefix ++ k, v)) -evalRelOp :: Database -> RelOp -> Either String [Row] -evalRelOp db (Scan mAlias tbl) = do +-- 索引查找的约定：主键列名是 "id"（优化器只在这种情况下才把过滤改写成 Lookup）。 +keyCondition :: Int -> Expr +keyCondition k = Eq (Col "id") (LitInt k) + +-- 扫描一张表；有别名就给每个列名加上前缀，JOIN 两边同名列靠它区分。 +evalScan :: Database -> Maybe String -> String -> Either String [Row] +evalScan db mAlias tbl = do table <- lookupTable db tbl let prefix = maybe "" (++ ".") mAlias Right (map (addPrefix prefix) (tableRows table)) + +-- 执行关系代数运算：Scan / Lookup / Filter / Project / Sort / Limit / Join 各一支。 +evalRelOp :: Database -> RelOp -> Either String [Row] +evalRelOp db (Scan mAlias tbl) = evalScan db mAlias tbl +evalRelOp db (Lookup tbl k) = do + rows <- evalScan db Nothing tbl + filterM (evalCondForRow (keyCondition k)) rows evalRelOp db (Filter e op) = do rows <- evalRelOp db op filterM (evalCondForRow e) rows @@ -37,3 +53,35 @@ evalRelOp db (Join left right cond) = do rrows <- evalRelOp db right let cross = [l ++ r | l <- lrows, r <- rrows] -- 笛卡尔积 filterM (evalCondForRow cond) cross + +-- | Monadic 版本：Lookup 直接问存储要一行（走索引），其余算子照原样递归。 +evalRelOpM :: (MonadStorage m) => Database -> RelOp -> m (Either String [Row]) +evalRelOpM _ (Lookup t k) = do + result <- lookupByKey t k + pure $ case result of + Left e -> Left e + Right Nothing -> Right [] + Right (Just r) -> Right [r] +evalRelOpM db (Scan a t) = pure (evalScan db a t) +evalRelOpM db (Filter p x) = do + result <- evalRelOpM db x + pure $ do + rows <- result + filterM (evalCondForRow p) rows +evalRelOpM db (Project cols x) = do + result <- evalRelOpM db x + pure (map (project cols) <$> result) +evalRelOpM db (Sort specs x) = do + result <- evalRelOpM db x + pure (sortRows specs <$> result) +evalRelOpM db (Limit n x) = do + result <- evalRelOpM db x + pure (take n <$> result) +evalRelOpM db (Join l r c) = do + ls <- evalRelOpM db l + rs <- evalRelOpM db r + pure $ do + leftRows <- ls + rightRows <- rs + let cross = [x ++ y | x <- leftRows, y <- rightRows] + filterM (evalCondForRow c) cross diff --git a/chusql-engine/src/ChuSQL/Algebra/Expr.hs b/chusql-engine/src/ChuSQL/Algebra/Expr.hs index 0fc7955..a47cbdf 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Expr.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Expr.hs @@ -4,6 +4,8 @@ import ChuSQL.Model import ChuSQL.Syntax.AST import Control.Monad (join) +-- 表达式求值：在一行（列名 → 值）上算出单个值；另外提供"静态收集表达式里用到哪些列"的工具。 + -- 表达式求值 evalExpr :: Expr -> Row -> Either String Value evalExpr (Col name) row = @@ -17,7 +19,7 @@ evalExpr (Eq a b) row = liftBinOp eqOp a b row evalExpr (And a b) row = liftBinOp (boolOp (&&)) a b row evalExpr (Or a b) row = liftBinOp (boolOp (||)) a b row --- | 收集表达式里用到的列名 +-- 收集表达式里用到的列名 colsInExpr :: Expr -> [String] colsInExpr (Col c) = [c] colsInExpr (Gt a b) = colsInExpr a ++ colsInExpr b diff --git a/chusql-engine/src/ChuSQL/Algebra/Op.hs b/chusql-engine/src/ChuSQL/Algebra/Op.hs index 77315af..bf4ab50 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Op.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Op.hs @@ -2,9 +2,12 @@ module ChuSQL.Algebra.Op (RelOp (..), renderPlan) where import ChuSQL.Syntax.AST (Expr, SortDir) +-- 关系代数算子：执行计划用 RelOp 表示；另外提供一个把计划打印成缩进文本的辅助函数。 + -- 关系代数算子 data RelOp = Scan (Maybe String) String + | Lookup String Int | Filter Expr RelOp | Project [String] RelOp | Sort [(String, SortDir)] RelOp @@ -12,11 +15,12 @@ data RelOp | Join RelOp RelOp Expr deriving (Show, Eq) --- | 把算子树画成缩进文本（演示和调试用） +-- 把算子树画成缩进文本（演示和调试用） renderPlan :: RelOp -> String renderPlan = go 0 where go ind op = case op of + Lookup t k -> "Lookup " ++ show t ++ " " ++ show k Scan a t -> pad ++ "Scan " ++ show a ++ " " ++ show t Filter e x -> pad ++ "Filter " ++ show e ++ "\n" ++ go (ind + 1) x Project c x -> pad ++ "Project " ++ show c ++ "\n" ++ go (ind + 1) x diff --git a/chusql-engine/src/ChuSQL/Algebra/Optimize.hs b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs index 10efcf2..76d4516 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Optimize.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Optimize.hs @@ -1,9 +1,11 @@ module ChuSQL.Algebra.Optimize (optimize, pushProject) where +import ChuSQL.Algebra.Expr (colsInExpr, evalExpr) import ChuSQL.Algebra.Op (RelOp (..)) import ChuSQL.Model import ChuSQL.Syntax.AST (Expr (..)) -import ChuSQL.Algebra.Expr (colsInExpr, evalExpr) + +-- 查询优化：谓词下推 + 投影下推 + 常量折叠，反复应用规则直到算子树形状不再变化。 -- * 谓词下推 @@ -15,6 +17,7 @@ relOpCols db (Limit _ x) = relOpCols db x relOpCols _ (Project ["*"] _) = ["*"] relOpCols _ (Project cols _) = cols relOpCols db (Join l r _) = relOpCols db l ++ relOpCols db r +relOpCols db (Lookup tbl _) = relOpCols db (Scan Nothing tbl) relOpCols db (Scan mAlias tbl) = case lookup tbl db of Nothing -> [] @@ -73,10 +76,18 @@ pushOne _ op = op -- * 投影下推 --- | 工具函数 +-- | 是不是要拿全部列（need 里含哨兵 "*"）。 needsAll :: [String] -> Bool needsAll = elem allColumns +-- | 叶子算子（扫描 / 索引查找）的投影下推：需要全部列就原样保留，缺列才套一层 Project +pushProjectLeaf :: Database -> [String] -> RelOp -> RelOp +pushProjectLeaf db need leaf + | needsAll need = leaf + | all (`elem` need) (relOpCols db leaf) = leaf + | otherwise = Project need leaf + +-- | 投影下推：把"上层需要哪些列"一路往下传，JOIN 两侧各自只保留自己需要的列。 pushProject :: Database -> [String] -> RelOp -> RelOp pushProject db _ (Project cols x) = case pushProject db cols x of Project cols' y | cols' == cols -> Project cols y @@ -93,10 +104,8 @@ pushProject db need (Join l r c) lNeed = if needsAll lCols then ["*"] else [x | x <- tot, x `elem` lCols] rNeed = if needsAll rCols then ["*"] else [x | x <- tot, x `elem` rCols] in Join (pushProject db lNeed l) (pushProject db rNeed r) c -pushProject db need scan@(Scan _ _) - | needsAll need = scan - | all (`elem` need) (relOpCols db scan) = scan - | otherwise = Project need scan +pushProject db need leaf@(Scan _ _) = pushProjectLeaf db need leaf +pushProject db need leaf@(Lookup _ _) = pushProjectLeaf db need leaf -- * 常量折叠 @@ -123,6 +132,8 @@ foldConstants e = e -- | 单节点重写：把 Filter 分派给上面几条规则，另外负责恒真条件删除与恒等投影消除 rewriteNode :: Database -> RelOp -> RelOp +rewriteNode _ (Filter (Eq (Col k) (LitInt v)) (Scan _ t)) + | k == "id" = Lookup t v rewriteNode db (Filter p (Join l r c)) = pushJoin db p l r c rewriteNode db (Filter p x) = case foldConstants p of LitBool True -> x diff --git a/chusql-engine/src/ChuSQL/Algebra/Planner.hs b/chusql-engine/src/ChuSQL/Algebra/Planner.hs index 8f645a2..909cd4b 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Planner.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Planner.hs @@ -3,12 +3,15 @@ module ChuSQL.Algebra.Planner (translate) where import ChuSQL.Algebra.Op import ChuSQL.Syntax.AST --- 内部查询语句转执行计划 +-- 计划翻译：把 AST 里的 SELECT 翻译成关系代数算子树（只翻译，不做合法性检查）。 + +-- 把 FROM 从句转成 Scan / Join 算子。 fromToRelOp :: FromClause -> RelOp fromToRelOp (FromTable mAlias tbl) = Scan mAlias tbl fromToRelOp (FromJoin left mAlias tbl cond) = Join (fromToRelOp left) (Scan mAlias tbl) cond +-- 翻译入口：把 SELECT 翻译成算子树；其它语句类型不支持（返回 Left）。 translate :: Query -> Either String RelOp translate q = case q of Select diff --git a/chusql-engine/src/ChuSQL/Algebra/Sort.hs b/chusql-engine/src/ChuSQL/Algebra/Sort.hs index 6e1bc08..1d9c471 100644 --- a/chusql-engine/src/ChuSQL/Algebra/Sort.hs +++ b/chusql-engine/src/ChuSQL/Algebra/Sort.hs @@ -4,20 +4,22 @@ import ChuSQL.Model import ChuSQL.Syntax.AST import Data.List (sortBy) --- | 反向排序 +-- 排序：按 ORDER BY 的列清单逐列比较（支持 ASC / DESC），供执行 Sort 算子时使用。 + +-- 反向排序 flipOrdering :: Ordering -> Ordering flipOrdering LT = GT flipOrdering GT = LT flipOrdering EQ = EQ --- | 定义值的比较规则 +-- 定义值的比较规则 compareValue :: Value -> Value -> Ordering compareValue (VInt a) (VInt b) = compare a b compareValue (VStr a) (VStr b) = compare a b compareValue (VBool a) (VBool b) = compare a b compareValue _ _ = EQ --- | 定义行的比较规则 +-- 定义行的比较规则 compareRows :: [(String, SortDir)] -> Row -> Row -> Ordering compareRows [] _ _ = EQ compareRows ((col, dir) : rest) r1 r2 = @@ -31,7 +33,7 @@ compareRows ((col, dir) : rest) r1 r2 = else o _ -> compareRows rest r1 r2 --- | 将表的每行排序 +-- 将表的每行排序 sortRows :: [(String, SortDir)] -> [Row] -> [Row] sortRows [] rows = rows sortRows spec rows = sortBy (compareRows spec) rows diff --git a/chusql-engine/src/ChuSQL/Engine.hs b/chusql-engine/src/ChuSQL/Engine.hs index 8f987dc..8215b18 100644 --- a/chusql-engine/src/ChuSQL/Engine.hs +++ b/chusql-engine/src/ChuSQL/Engine.hs @@ -1,78 +1,100 @@ -module ChuSQL.Engine (runQuery, rowsOf) where +module ChuSQL.Engine (runQuery, runQueryM, rowsOf) where -import ChuSQL.Algebra.Eval (evalRelOp) +import ChuSQL.Algebra.Eval (evalRelOpM) import ChuSQL.Algebra.Expr (evalCondForRow, evalExpr) import ChuSQL.Algebra.Optimize (optimize) import ChuSQL.Algebra.Planner (translate) import ChuSQL.Model import ChuSQL.Semantic (check) +import ChuSQL.Storage (MonadStorage (..)) +import ChuSQL.Storage.Memory (MemoryStorage (runMemoryStorage)) import ChuSQL.Syntax.AST import Control.Monad (filterM) --- | 带检查的查询入口：先做语义检查，再交给下面的执行 +-- | 对外入口：签名不变，内部走内存实现。 runQuery :: Database -> Query -> Either String (Database, [Row]) runQuery db q = do - _ <- check db q - runQueryUnchecked db q + (result, db') <- runMemoryStorage (runQueryM q) db + rows <- result + Right (db', rows) --- | 只要结果行（丢掉更新后的数据库），给演示和测试用 +-- | 只要结果行（丢掉更新后的数据库），给演示和测试用。 rowsOf :: Either String (Database, [Row]) -> Either String [Row] rowsOf = fmap snd --- 把表整体换掉（调用方已经查过，表一定存在） -replaceTable :: String -> Table -> Database -> Database -replaceTable name table = map (\(n, t) -> if n == name then (n, table) else (n, t)) +-- * 泛型入口：走 MonadStorage，未来 IPCStorage 复用同一份代码 --- 修改表 -applyUpdates :: [(String, Expr)] -> Row -> Either String Row -applyUpdates [] row = Right row -applyUpdates ((col, e) : rest) row = do - v <- evalExpr e row - let row' = setColumn col v row - applyUpdates rest row' +-- | 语义检查 + 分发到具体语句。 +runQueryM :: (MonadStorage m) => Query -> m (Either String [Row]) +runQueryM q = do + db <- snapshot + case check db q of + Left err -> pure (Left err) + Right _ -> runQueryUncheckedM q + +-- | SELECT：拿快照，跑 translate → optimize → evalRelOp。 +runQueryUncheckedM :: (MonadStorage m) => Query -> m (Either String [Row]) +runQueryUncheckedM q@Select{} = do + db <- snapshot + case translate q of + Left e -> pure (Left e) + Right relOp -> evalRelOpM db (optimize db relOp) + +-- \| INSERT：求值后追加一行。 +runQueryUncheckedM (Insert tbl cols vals) = + case mapM (\e -> evalExpr e []) vals of + Left err -> pure (Left err) + Right values + | length cols /= length values -> + pure (Left "column count does not match value count") + | otherwise -> do + result <- insert tbl (zip cols values) + pure (result >> Right []) + +-- \| DELETE：扫表、筛掉命中行、整体写回。 +runQueryUncheckedM (Delete tbl mWhere) = do + rowsResult <- scan tbl + case rowsResult of + Left err -> pure (Left err) + Right rows -> do + let kept = case mWhere of + Nothing -> Right [] + Just e -> filterM (shouldKeep e) rows + case kept of + Left err -> pure (Left err) + Right keptRows -> do + result <- replaceAll tbl keptRows + pure (result >> Right []) where - setColumn c v r = map (\(k, val) -> if k == c then (k, v) else (k, val)) r - --- 实际执行（调用方请确保已过 check） -runQueryUnchecked :: Database -> Query -> Either String (Database, [Row]) -runQueryUnchecked db q@Select{} = do - relOp <- translate q - rows <- evalRelOp db (optimize db relOp) - Right (db, rows) - --- 处理插入 -runQueryUnchecked db (Insert tbl cols vals) = do - values <- mapM (\e -> evalExpr e []) vals - if length cols /= length values - then Left "column count does not match value count" - else do - table <- lookupTable db tbl - let newRow = zip cols values - Right (replaceTable tbl table{tableRows = tableRows table ++ [newRow]} db, []) - --- 处理删除 -runQueryUnchecked db (Delete tbl mWhere) = do - table <- lookupTable db tbl - rows <- case mWhere of - Nothing -> Right [] - Just e -> filterM (shouldKeep e) (tableRows table) - Right (replaceTable tbl table{tableRows = rows} db, []) - where - -- 保留 = 条件不成立 + -- \| 保留 = 条件不成立。 shouldKeep :: Expr -> Row -> Either String Bool shouldKeep e row = not <$> evalCondForRow e row --- 处理表更新 -runQueryUnchecked db (Update tbl assigns mWhere) = do - table <- lookupTable db tbl - rows <- mapM (updateRow mWhere assigns) (tableRows table) - Right (replaceTable tbl table{tableRows = rows} db, []) +-- \| UPDATE：扫表、逐行改、整体写回。 +runQueryUncheckedM (Update tbl assigns mWhere) = do + rowsResult <- scan tbl + case rowsResult of + Left err -> pure (Left err) + Right rows -> do + let newRows = mapM (updateRow mWhere assigns) rows + case newRows of + Left err -> pure (Left err) + Right rs -> do + result <- replaceAll tbl rs + pure (result >> Right []) where + -- \| 命中条件就套用赋值，否则原样返回。 updateRow :: Maybe Expr -> [(String, Expr)] -> Row -> Either String Row updateRow cond asgns row = do keep <- case cond of Nothing -> Right True Just e -> evalCondForRow e row - if keep - then applyUpdates asgns row - else Right row + if keep then applyUpdates asgns row else Right row + +-- | 按 assignments 依次求值并覆盖对应列。 +applyUpdates :: [(String, Expr)] -> Row -> Either String Row +applyUpdates [] row = Right row +applyUpdates ((col, e) : rest) row = do + v <- evalExpr e row + let row' = map (\(k, val) -> if k == col then (k, v) else (k, val)) row + applyUpdates rest row' diff --git a/chusql-engine/src/ChuSQL/Model.hs b/chusql-engine/src/ChuSQL/Model.hs index 6b11f30..0d0be17 100644 --- a/chusql-engine/src/ChuSQL/Model.hs +++ b/chusql-engine/src/ChuSQL/Model.hs @@ -1,5 +1,7 @@ module ChuSQL.Model (Column (..), Value (..), Row, Table (..), Database, colNames, colType, lookupTable, allColumns) where +-- 数据模型：列类型、值、行、表、数据库，外加"按名查表 / 查列类型"这类通用小工具。 + -- 列的类型 data Column = TInt @@ -32,14 +34,14 @@ type Database = [(String, Table)] colNames :: Table -> [String] colNames = map fst . tableCols --- | 查某一列的类型 +-- 查某一列的类型 colType :: Table -> String -> Maybe Column colType t c = lookup c (tableCols t) --- | 按名字查表；表不存在就报错 +-- 按名字查表；表不存在就报错 lookupTable :: Database -> String -> Either String Table lookupTable db t = maybe (Left ("unknown table: " ++ t)) Right (lookup t db) --- | 投影列清单里的哨兵值：它表示"全部列，不用挑"，不是列名 +-- 投影列清单里的哨兵值：它表示"全部列，不用挑"，不是列名 allColumns :: String allColumns = "*" diff --git a/chusql-engine/src/ChuSQL/Semantic.hs b/chusql-engine/src/ChuSQL/Semantic.hs index d65c7b0..d5c5d18 100644 --- a/chusql-engine/src/ChuSQL/Semantic.hs +++ b/chusql-engine/src/ChuSQL/Semantic.hs @@ -5,6 +5,8 @@ import ChuSQL.Model import ChuSQL.Syntax.AST import Data.List (intercalate) +-- 语义检查：执行前的关卡 —— 表和列存不存在、表达式类型对不对、写入的值类型是否匹配。 + -- * 表与列名解析 -- | 给一批列加上"别名."前缀 diff --git a/chusql-engine/src/ChuSQL/Storage.hs b/chusql-engine/src/ChuSQL/Storage.hs new file mode 100644 index 0000000..87435bd --- /dev/null +++ b/chusql-engine/src/ChuSQL/Storage.hs @@ -0,0 +1,20 @@ +module ChuSQL.Storage (MonadStorage (..)) where + +import ChuSQL.Model (Database, Row) + +-- 存储层抽象：上层只认 domain 类型，不关心底层是内存还是 IPC。 +class (Monad m) => MonadStorage m where + -- 全表扫描；表不存在返回 Left。 + scan :: String -> m (Either String [Row]) + + -- 追加一行到表尾；表不存在返回 Left。 + insert :: String -> Row -> m (Either String ()) + + -- 用新行列表整体替换一张表。 + replaceAll :: String -> [Row] -> m (Either String ()) + + -- 按主键查一行；命中返回 Just，未命中返回 Nothing。 + lookupByKey :: String -> Int -> m (Either String (Maybe Row)) + + -- 取全库快照，喂给还接收 Database 的语义检查 / 优化器 / evalRelOp。 + snapshot :: m Database diff --git a/chusql-engine/src/ChuSQL/Storage/.gitkeep b/chusql-engine/src/ChuSQL/Storage/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/chusql-engine/src/ChuSQL/Storage/IPC.hs b/chusql-engine/src/ChuSQL/Storage/IPC.hs new file mode 100644 index 0000000..3a89c41 --- /dev/null +++ b/chusql-engine/src/ChuSQL/Storage/IPC.hs @@ -0,0 +1,331 @@ +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE OverloadedStrings #-} + +module ChuSQL.Storage.IPC ( + IPCStorage (..), + Request (..), + Response (..), + SchemaColumn (..), + sendRequest, + doListTables, + doLookupByKey, + doInsertKeyed, + doDescribeTable, +) where + +import ChuSQL.Model +import ChuSQL.Storage +import Control.Exception (bracket) +import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecodeStrict, encode, object, withObject, (.:), (.=)) +import Data.Aeson qualified as A +import Data.Aeson.Key qualified as K +import Data.Aeson.KeyMap qualified as KM +import Data.Aeson.Types (Parser) +import Data.ByteString.Char8 qualified as BSC +import Data.ByteString.Lazy qualified as BL +import Data.List (nub) +import Data.Scientific (floatingOrInteger) +import Data.Text qualified as T +import System.Environment (lookupEnv) +import System.IO + +-- IPC 存储实现：每个 MonadStorage 方法翻译成管道上的一条 JSON 请求。 + +-- | IPC 实现：通过命名管道连 Rust 存储进程。 +newtype IPCStorage a = IPCStorage + { runIPCStorage :: IO a + } + +-- | Functor：把 fmap 转发给底下的 IO。 +instance Functor IPCStorage where + fmap f (IPCStorage m) = IPCStorage (fmap f m) + +-- | Applicative：把 pure / <*> 转发给底下的 IO。 +instance Applicative IPCStorage where + pure = IPCStorage . pure + IPCStorage mf <*> IPCStorage ma = IPCStorage (mf <*> ma) + +-- | Monad：顺序执行，转发给底下的 IO。 +instance Monad IPCStorage where + IPCStorage m >>= k = IPCStorage $ do + a <- m + runIPCStorage (k a) + +-- | 管道路径；默认 \\.\pipe\chusql-storage，可用 CHUSQL_PIPE 覆盖。 +pipePath :: IO String +pipePath = do + name <- maybe "chusql-storage" id <$> lookupEnv "CHUSQL_PIPE" + pure ("\\\\.\\pipe\\" ++ name) + +-- | 打开管道、跑闭包、关闭。 +withPipe :: (Handle -> IO a) -> IO a +withPipe = bracket openPipe hClose + where + -- \| 二进制读写、不做换行翻译，跟 Rust 侧对齐。 + openPipe = do + path <- pipePath + h <- openFile path ReadWriteMode + hSetBuffering h LineBuffering + hSetNewlineMode h noNewlineTranslation + pure h + +-- | 发一条请求，读一条响应。 +sendRequest :: Request -> IO Response +sendRequest req = withPipe $ \h -> do + BL.hPutStr h (encode req) + BSC.hPutStr h "\n" + hFlush h + line <- BSC.hGetLine h + let cleaned = BSC.dropWhileEnd (== '\r') line + case eitherDecodeStrict cleaned of + Left err -> pure (RespError ("decode: " ++ err)) + Right r -> pure r + +-- * JSON 编解码 + +-- | 一列的 schema；对应 Rust 侧的 SchemaColumn。 +data SchemaColumn = SchemaColumn + { scName :: String + , scType :: String + } + deriving (Show, Eq) + +-- | SchemaColumn 编码；Haskell 侧只发 DescribeTable，编码用不上，暂存实现。 +instance ToJSON SchemaColumn where + toJSON (SchemaColumn n t) = object ["name" .= n, "ty" .= t] + +-- | SchemaColumn 解码：从 {"name": "...", "ty": "int"|"str"|"bool"} 读回。 +instance FromJSON SchemaColumn where + parseJSON = withObject "SchemaColumn" $ \o -> do + n <- o .: "name" + t <- o .: "ty" + pure (SchemaColumn n t) + +-- | 请求；对应 Rust 侧带 method tag 的 enum。 +data Request + = ReqPing + | ReqScan String + | ReqInsert String Row (Maybe Int) + | ReqReplaceAll String [Row] + | ReqListTables + | ReqLookupByKey String Int + | ReqDescribeTable String + +-- | 请求编码：method 字段 + 各自参数。 +instance ToJSON Request where + toJSON ReqPing = object ["method" .= ("ping" :: T.Text)] + toJSON (ReqScan t) = object ["method" .= ("scan" :: T.Text), "table" .= t] + toJSON (ReqInsert t r mk) = + object $ + [ "method" .= ("insert" :: T.Text) + , "table" .= t + , "row" .= rowToJSON r + ] + ++ case mk of + Nothing -> [] + Just k -> ["key" .= k] + toJSON (ReqReplaceAll t rs) = + object + [ "method" .= ("replace_all" :: T.Text) + , "table" .= t + , "rows" .= map rowToJSON rs + ] + toJSON ReqListTables = object ["method" .= ("list_tables" :: T.Text)] + toJSON (ReqLookupByKey t k) = + object + [ "method" .= ("lookup_by_index" :: T.Text) + , "table" .= t + , "key" .= k + ] + toJSON (ReqDescribeTable t) = + object + [ "method" .= ("describe_table" :: T.Text) + , "table" .= t + ] + +-- | 响应；对应 Rust 侧带 status tag 的 enum。 +data Response + = RespPong + | RespRows [Row] + | RespTables [String] + | RespSchema [SchemaColumn] + | RespOk + | RespError String + +-- | 响应解码：按 status 字段分派。 +instance FromJSON Response where + parseJSON = withObject "Response" $ \o -> do + status <- o .: "status" + case status :: T.Text of + "pong" -> pure RespPong + "ok" -> pure RespOk + "rows" -> RespRows <$> (o .: "rows" >>= mapM rowFromJSON) + "tables" -> RespTables <$> o .: "tables" + "schema" -> RespSchema <$> o .: "columns" + "error" -> RespError <$> o .: "message" + other -> fail ("unknown status: " ++ T.unpack other) + +-- | Value 编码成 JSON 的 number / string / bool。 +valueToJSON :: Value -> A.Value +valueToJSON (VInt n) = A.Number (fromIntegral n) +valueToJSON (VStr s) = A.String (T.pack s) +valueToJSON (VBool b) = A.Bool b + +-- | JSON 解回 Value；number 只接受整数。 +valueFromJSON :: A.Value -> Parser Value +valueFromJSON (A.Number n) = + case floatingOrInteger n :: Either Double Integer of + Right i -> pure (VInt (fromIntegral i)) + Left _ -> fail "non-integer number" +valueFromJSON (A.String s) = pure (VStr (T.unpack s)) +valueFromJSON (A.Bool b) = pure (VBool b) +valueFromJSON _ = fail "unsupported value type" + +-- | 一行编成 JSON object。 +rowToJSON :: Row -> A.Value +rowToJSON r = + A.Object (KM.fromList [(K.fromString k, valueToJSON v) | (k, v) <- r]) + +-- | JSON object 解成一行。 +rowFromJSON :: A.Value -> Parser Row +rowFromJSON (A.Object o) = mapM toPair (KM.toList o) + where + -- \| 解一个键值对。 + toPair (k, v) = do + val <- valueFromJSON v + pure (K.toString k, val) +rowFromJSON _ = fail "row must be a JSON object" + +-- * 内部请求封装 + +-- | 发 Scan 并把结果取成 Either。 +doScan :: String -> IO (Either String [Row]) +doScan t = do + resp <- sendRequest (ReqScan t) + pure $ case resp of + RespRows rows -> Right rows + RespError e -> Left e + _ -> Left "unexpected response to scan" + +-- | 发 Insert（不带索引键）。 +doInsert :: String -> Row -> IO (Either String ()) +doInsert t r = do + resp <- sendRequest (ReqInsert t r Nothing) + pure $ case resp of + RespOk -> Right () + RespError e -> Left e + _ -> Left "unexpected response to insert" + +-- | 发 Insert（带索引键）。 +doInsertKeyed :: String -> Row -> Int -> IO (Either String ()) +doInsertKeyed t r k = do + resp <- sendRequest (ReqInsert t r (Just k)) + pure $ case resp of + RespOk -> Right () + RespError e -> Left e + _ -> Left "unexpected response to insert" + +-- | 发 ReplaceAll。 +doReplaceAll :: String -> [Row] -> IO (Either String ()) +doReplaceAll t rs = do + resp <- sendRequest (ReqReplaceAll t rs) + pure $ case resp of + RespOk -> Right () + RespError e -> Left e + _ -> Left "unexpected response to replace_all" + +-- | 发 ListTables。 +doListTables :: IO (Either String [String]) +doListTables = do + resp <- sendRequest ReqListTables + pure $ case resp of + RespTables ts -> Right ts + RespError e -> Left e + _ -> Left "unexpected response to list_tables" + +-- | 发 LookupByIndex。 +doLookupByKey :: String -> Int -> IO (Either String (Maybe Row)) +doLookupByKey t k = do + resp <- sendRequest (ReqLookupByKey t k) + pure $ case resp of + RespRows [] -> Right Nothing + RespRows (r : _) -> Right (Just r) + RespError e -> Left e + _ -> Left "unexpected response to lookup_by_index" + +-- | 发 DescribeTable。 +doDescribeTable :: String -> IO (Either String [SchemaColumn]) +doDescribeTable t = do + resp <- sendRequest (ReqDescribeTable t) + pure $ case resp of + RespSchema cols -> Right cols + RespError e -> Left e + _ -> Left "unexpected response to describe_table" + +-- | 把 Rust 的 schema 列转成 Model 的 Column；未知类型降级为 TStr。 +schemaToColumns :: [SchemaColumn] -> [(String, Column)] +schemaToColumns = map go + where + -- \| 按 ty 字符串映射成 Column 构造子。 + go (SchemaColumn n "int") = (n, TInt) + go (SchemaColumn n "str") = (n, TStr) + go (SchemaColumn n "bool") = (n, TBool) + go (SchemaColumn n _) = (n, TStr) + +-- | 从若干行推断列名与类型；catalog 拿不到时的兜底。 +inferColumns :: [Row] -> [(String, Column)] +inferColumns rows = [(n, inferType n) | n <- names] + where + -- \| 全部出现过的列名。 + names = nub (concatMap (map fst) rows) + + -- \| 用第一个出现的值推断列类型；找不到就给 TStr。 + inferType n = case [v | r <- rows, Just v <- [lookup n r]] of + (VInt _ : _) -> TInt + (VStr _ : _) -> TStr + (VBool _ : _) -> TBool + [] -> TStr + +-- | 先 describe 拿列，再 scan 拿行；describe 拿不到时退回扫行推断。 +loadTable :: String -> IO Table +loadTable t = do + schemaR <- doDescribeTable t + case schemaR of + Right cols -> do + rowsR <- doScan t + case rowsR of + Right rows -> pure (Table t (schemaToColumns cols) rows) + Left _ -> pure (Table t (schemaToColumns cols) []) + Left _ -> do + rowsR <- doScan t + case rowsR of + Right rows -> pure (Table t (inferColumns rows) rows) + Left _ -> pure (Table t [] []) + +-- * MonadStorage 实例 + +-- | IPC 实现：所有方法都走管道。 +instance MonadStorage IPCStorage where + -- \| 发 Scan。 + scan t = IPCStorage (doScan t) + + -- \| 发 Insert，不带索引键。 + insert t r = IPCStorage (doInsert t r) + + -- \| 发 ReplaceAll。 + replaceAll t rs = IPCStorage (doReplaceAll t rs) + + -- \| 发 LookupByIndex。 + lookupByKey t k = IPCStorage (doLookupByKey t k) + + -- \| ListTables + 逐表 (describe, scan)，拼成 Database。 + snapshot = IPCStorage $ do + result <- doListTables + case result of + Left _ -> pure [] + Right ts -> mapM loadDbEntry ts + where + -- \| 加载一个 (表名, 表) 对。 + loadDbEntry t = do + tbl <- loadTable t + pure (t, tbl) diff --git a/chusql-engine/src/ChuSQL/Storage/Memory.hs b/chusql-engine/src/ChuSQL/Storage/Memory.hs new file mode 100644 index 0000000..03e5332 --- /dev/null +++ b/chusql-engine/src/ChuSQL/Storage/Memory.hs @@ -0,0 +1,66 @@ +module ChuSQL.Storage.Memory (MemoryStorage (runMemoryStorage)) where + +import ChuSQL.Model +import ChuSQL.Storage + +-- | 内存实现：本质是 Database 上的 State，错误通道是 Either String。 +newtype MemoryStorage a = MemoryStorage {runMemoryStorage :: Database -> Either String (a, Database)} + +-- | Functor：把 fmap 转发给底下的 State。 +instance Functor MemoryStorage where + fmap f (MemoryStorage m) = MemoryStorage $ \db -> do + (a, db') <- m db + Right (f a, db') + +-- | Applicative：转发 pure / <*>。 +instance Applicative MemoryStorage where + pure x = MemoryStorage $ \db -> Right (x, db) + MemoryStorage mf <*> MemoryStorage ma = MemoryStorage $ \db -> do + (f, db1) <- mf db + (a, db2) <- ma db1 + Right (f a, db2) + +-- | Monad：顺序执行，把动作转发给底下的 State。 +instance Monad MemoryStorage where + MemoryStorage m >>= k = MemoryStorage $ \db -> do + (a, db') <- m db + runMemoryStorage (k a) db' + +-- | MonadStorage 实例：内存里直接改 Database。 +instance MonadStorage MemoryStorage where + -- \| 按名查表，返回所有行。 + scan t = MemoryStorage $ \db -> do + tbl <- lookupTable db t + Right (Right (tableRows tbl), db) + + -- \| 追加一行到表尾。 + insert t r = MemoryStorage $ \db -> do + tbl <- lookupTable db t + Right (Right (), replaceTable t tbl{tableRows = tableRows tbl ++ [r]} db) + + -- \| 用新行列表替换整表内容。 + replaceAll t rows = MemoryStorage $ \db -> do + tbl <- lookupTable db t + Right (Right (), replaceTable t tbl{tableRows = rows} db) + + -- \| 找 id 列等于 k 的第一行。 + lookupByKey t k = MemoryStorage $ \db -> do + tbl <- lookupTable db t + Right (Right (findRow tbl k), db) + + -- \| 原样返回当前数据库。 + snapshot = MemoryStorage $ \db -> Right (db, db) + +-- | 按 id 列匹配一个整数。 +findRow :: Table -> Int -> Maybe Row +findRow tbl k = case filter matches (tableRows tbl) of + (r : _) -> Just r + [] -> Nothing + where + -- \| 这行的 id 列是否等于 k。 + matches r = lookup "id" r == Just (VInt k) + +-- | 用给定表替换同名表；不存在就原样返回。 +replaceTable :: String -> Table -> Database -> Database +replaceTable name table = + map (\(n, t) -> if n == name then (n, table) else (n, t)) diff --git a/chusql-engine/src/ChuSQL/Syntax/AST.hs b/chusql-engine/src/ChuSQL/Syntax/AST.hs index dedfea8..a4f57e6 100644 --- a/chusql-engine/src/ChuSQL/Syntax/AST.hs +++ b/chusql-engine/src/ChuSQL/Syntax/AST.hs @@ -1,5 +1,7 @@ module ChuSQL.Syntax.AST (Query (..), FromClause (..), Expr (..), SortDir (..), makeSelect) where +-- SQL 语法树：四类语句（SELECT / INSERT / DELETE / UPDATE）+ 表达式、FROM 从句、排序方向。 + -- 查询语句 data Query = Select @@ -14,7 +16,7 @@ data Query | Update String [(String, Expr)] (Maybe Expr) deriving (Show, Eq) --- 从句 +-- FROM 从句：单表或 JOIN。 data FromClause = FromTable (Maybe String) String | FromJoin FromClause (Maybe String) String Expr @@ -33,11 +35,13 @@ data Expr | Or Expr Expr deriving (Show, Eq) +-- 排序方向 data SortDir = Asc | Desc deriving (Show, Eq) +-- 便捷构造：造一条最简单的 SELECT（给测试和演示用）。 makeSelect :: [String] -> String -> Maybe Expr -> Query makeSelect cols tbl w = Select diff --git a/chusql-engine/src/ChuSQL/Syntax/Parser.hs b/chusql-engine/src/ChuSQL/Syntax/Parser.hs index 6e44ac1..9b5cd77 100644 --- a/chusql-engine/src/ChuSQL/Syntax/Parser.hs +++ b/chusql-engine/src/ChuSQL/Syntax/Parser.hs @@ -1,10 +1,5 @@ {-# LANGUAGE ImportQualifiedPost #-} -{- | SQL 语法分析器：把 SQL 文本解析为 'ChuSQL.Syntax.AST.Query'。 - -词法层负责空白、关键字、标识符、字面量与符号；语法层按 -优先级 比较 > AND > OR 自底向上组装表达式。 --} module ChuSQL.Syntax.Parser (parseQuery) where import ChuSQL.Syntax.AST @@ -17,6 +12,9 @@ import Text.Megaparsec import Text.Megaparsec.Char import Text.Megaparsec.Char.Lexer qualified as Lxr +-- SQL 解析器：把 SQL 文本解析成 AST。词法层管空白、关键字、标识符、字面量与符号； +-- 语法层按优先级 比较 > AND > OR 自底向上组装表达式。 + type Parser = Parsec Void String -- * 词法层 @@ -29,12 +27,12 @@ sc = Lxr.space space1 empty empty lexeme :: Parser a -> Parser a lexeme = Lxr.lexeme sc +-- | 匹配一个符号（如 ","、"("），并吃掉其后的空白。 symbol :: String -> Parser String symbol = Lxr.symbol sc -{- | 大小写不敏感的关键字；其后不能紧跟标识符字符，避免把 @selection@ 读成 @select@。 -整体用 'try' 包住：否则 @ORDER@ 会被 @keyword "or"@ 吃掉前缀 @OR@ 再报错，导致无法回溯。 --} +-- | 大小写不敏感的关键字；其后不能紧跟标识符字符，避免把 @selection@ 读成 @select@。 +-- 整体用 'try' 包住：否则 @ORDER@ 会被 @keyword "or"@ 吃掉前缀 @OR@ 再报错，导致无法回溯。 keyword :: String -> Parser () keyword k = lexeme $ try $ do _ <- string' k @@ -47,6 +45,7 @@ identifier = lexeme $ do rest <- many (satisfy isIdentChar) return (first : rest) +-- | 标识符允许的字符：字母、数字或下划线。 isIdentChar :: Char -> Bool isIdentChar ch = isAlphaNum ch || ch == '_' @@ -62,9 +61,8 @@ stringLit = lexeme $ do _ <- char '\'' return s -{- | 字符串内的一个字符：@''@ 折叠成一个单引号，其余字符（含反斜杠）原样保留。 -两个分支都不会在失败时消耗输入，因此可以安全地放进 'many'。 --} +-- | 字符串内的一个字符：@''@ 折叠成一个单引号，其余字符（含反斜杠）原样保留。 +-- 两个分支都不会在失败时消耗输入，因此可以安全地放进 'many'。 stringChar :: Parser Char stringChar = choice @@ -95,9 +93,8 @@ qualifiedName = do expr :: Parser Expr expr = makeExprParser atom operatorTable -{- | 运算符表。注意 'makeExprParser' 要求各层 **按优先级从高到低** 排列： -比较运算符绑定最紧，'AND' 次之，'OR' 最松。 --} +-- | 运算符表。注意 'makeExprParser' 要求各层按优先级从高到低排列： +-- 比较运算符绑定最紧，'AND' 次之，'OR' 最松。 operatorTable :: [[Operator Parser Expr]] operatorTable = [ @@ -128,12 +125,14 @@ orderItem = do dir <- sortDir return (col, dir) +-- | @ORDER BY col [ASC|DESC], ...@。 orderByClause :: Parser [(String, SortDir)] orderByClause = do keyword "order" keyword "by" sepBy1 orderItem (symbol ",") +-- | @col = expr@，用于 UPDATE 的 SET 子句。 assignment :: Parser (String, Expr) assignment = do col <- identifier @@ -141,7 +140,7 @@ assignment = do e <- expr return (col, e) --- | 从句语法 +-- * 从句层 -- | SQL 保留字，不能用作表别名 reservedWords :: [String] @@ -166,21 +165,22 @@ reservedWords = , "set" ] -{- | 表别名。别名不能是保留字：否则 @FROM users WHERE age > 18@ 会把 @WHERE@ -当成 @users@ 的别名吃掉，后面真正的 WHERE 子句就再也解析不到。 -用 'try' 包住，一旦命中保留字就整体回溯，让 'optional' 正常返回 'Nothing'。 --} +-- | 表别名。别名不能是保留字：否则 @FROM users WHERE age > 18@ 会把 @WHERE@ +-- 当成 @users@ 的别名吃掉，后面真正的 WHERE 子句就再也解析不到。 +-- 用 'try' 包住，一旦命中保留字就整体回溯，让 'optional' 正常返回 'Nothing'。 aliasName :: Parser String aliasName = try $ do name <- identifier if map toLower name `elem` reservedWords then empty else return name +-- | @表名 [别名]@。 tableRef :: Parser (Maybe String, String) tableRef = do tbl <- identifier mAlias <- optional aliasName return (mAlias, tbl) +-- | @JOIN 表 [别名] ON 条件@。 joinClause :: Parser (Maybe String, String, Expr) joinClause = do keyword "join" @@ -189,6 +189,7 @@ joinClause = do cond <- expr return (mAlias, tbl, cond) +-- | FROM 从句：一张表 + 任意多个 JOIN。 fromClause :: Parser FromClause fromClause = do (mAlias, tbl) <- tableRef diff --git a/chusql-engine/test/Spec.hs b/chusql-engine/test/Spec.hs index 619b9c2..06a3415 100644 --- a/chusql-engine/test/Spec.hs +++ b/chusql-engine/test/Spec.hs @@ -4,15 +4,24 @@ import ChuSQL.Algebra.Eval (evalRelOp) import ChuSQL.Algebra.Op (RelOp (..), renderPlan) import ChuSQL.Algebra.Optimize (optimize, pushProject) import ChuSQL.Algebra.Planner (translate) -import ChuSQL.Model -import ChuSQL.Syntax.AST import ChuSQL.Engine +import ChuSQL.Model +import ChuSQL.Storage (MonadStorage (..)) +import ChuSQL.Storage.IPC (IPCStorage (runIPCStorage), Request (ReqPing), Response (RespPong), doListTables, sendRequest) +import ChuSQL.Syntax.AST import ChuSQL.Syntax.Parser +import Control.Concurrent (threadDelay) +import Control.Exception (IOException, bracket, try) +import Data.List (isInfixOf, sortOn) +import System.CPUTime (getCPUTime) +import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist, getTemporaryDirectory, removePathForcibly) +import System.Environment (getEnvironment, lookupEnv, setEnv, unsetEnv) +import System.Exit (ExitCode (..)) +import System.FilePath (()) +import System.IO (Handle) +import System.Process (CreateProcess (..), ProcessHandle, StdStream (NoStream), createProcess, proc, terminateProcess, waitForProcess) import Test.Hspec --- ============================================================ --- Test database --- ============================================================ users :: Table users = Table @@ -40,14 +49,8 @@ orders = testDB :: Database testDB = [("users", users), ("orders", orders)] --- ============================================================ --- Main --- ============================================================ main :: IO () main = hspec $ do - -- ============================================================ - -- AST - -- ============================================================ describe "ChuSQL.Syntax.AST" $ do it "equal Selects are equal" $ do makeSelect ["name"] "users" Nothing @@ -57,9 +60,6 @@ main = hspec $ do makeSelect ["name"] "users" Nothing `shouldNotBe` makeSelect ["age"] "users" Nothing - -- ============================================================ - -- Parser - -- ============================================================ describe "ChuSQL.Syntax.Parser" $ do it "parses a simple SELECT" $ do parseQuery "SELECT name FROM users" @@ -201,9 +201,6 @@ main = hspec $ do parseQuery "SELECT name FROM" `shouldSatisfy` isLeft - -- ============================================================ - -- Parser: JOIN - -- ============================================================ describe "ChuSQL.Syntax.Parser (JOIN)" $ do it "parses a simple JOIN without aliases" $ do parseQuery "SELECT name FROM users JOIN orders ON id = user_id" @@ -244,9 +241,6 @@ main = hspec $ do Right q -> selectCols q `shouldBe` ["u.name", "o.product"] Left err -> expectationFailure err - -- ============================================================ - -- Engine: SELECT - -- ============================================================ describe "ChuSQL.Engine (SELECT)" $ do it "returns all rows without WHERE" $ do rowsOf (runQuery testDB (makeSelect ["*"] "users" Nothing)) @@ -301,9 +295,6 @@ main = hspec $ do runQuery testDB (makeSelect ["nope"] "users" Nothing) `shouldSatisfy` isLeft - -- ============================================================ - -- Engine: INSERT - -- ============================================================ describe "ChuSQL.Engine (INSERT)" $ do it "parses a simple INSERT" $ do parseQuery "INSERT INTO users (name, age) VALUES ('Dave', 22)" @@ -350,9 +341,6 @@ main = hspec $ do runQuery testDB (Insert "users" ["name"] [Col "other"]) `shouldSatisfy` isLeft - -- ============================================================ - -- Engine: DELETE - -- ============================================================ describe "ChuSQL.Engine (DELETE)" $ do it "parses DELETE with WHERE" $ do parseQuery "DELETE FROM users WHERE age < 18" @@ -400,9 +388,6 @@ main = hspec $ do runQuery testDB (Delete "users" (Just (Gt (Col "unknown") (LitInt 18)))) `shouldSatisfy` isLeft - -- ============================================================ - -- Engine: UPDATE - -- ============================================================ describe "ChuSQL.Engine (UPDATE)" $ do it "parses UPDATE with WHERE" $ do parseQuery "UPDATE users SET age = 26 WHERE name = 'Alice'" @@ -492,9 +477,6 @@ main = hspec $ do Right _ -> True Left _ -> True - -- ============================================================ - -- Engine: ORDER BY - -- ============================================================ describe "ChuSQL.Engine (ORDER BY)" $ do it "parses ORDER BY ASC" $ do case parseQuery "SELECT name FROM users ORDER BY age ASC" of @@ -580,9 +562,6 @@ main = hspec $ do } runQuery testDB q `shouldSatisfy` isLeft - -- ============================================================ - -- Engine: LIMIT - -- ============================================================ describe "ChuSQL.Engine (LIMIT)" $ do it "parses LIMIT" $ do case parseQuery "SELECT * FROM users LIMIT 2" of @@ -663,9 +642,6 @@ main = hspec $ do parseQuery "SELECT * FROM users LIMIT -1" `shouldSatisfy` isLeft - -- ============================================================ - -- Engine: JOIN - -- ============================================================ describe "ChuSQL.Engine (JOIN)" $ do it "executes a two-table JOIN with aliases" $ do rowsOf (parseQuery "SELECT u.name, o.product FROM users u JOIN orders o ON u.id = o.user_id" >>= runQuery testDB) @@ -733,9 +709,6 @@ main = hspec $ do ) `shouldSatisfy` isLeft - -- ============================================================ - -- End-to-end - -- ============================================================ describe "ChuSQL end-to-end" $ do it "parses and executes a simple query" $ do rowsOf (parseQuery "SELECT name FROM users WHERE age > 18" >>= runQuery testDB) @@ -937,7 +910,8 @@ main = hspec $ do let plan = Project ["name"] - ( Limit 2 + ( Limit + 2 ( Sort [("age", Desc)] (Filter (Gt (Col "age") (LitInt 18)) (Scan Nothing "users")) @@ -979,10 +953,12 @@ main = hspec $ do it "keeps projection with ORDER BY and LIMIT results identical" $ do sameResultAsUnoptimized "SELECT u.name FROM users u JOIN orders o ON u.id = o.user_id ORDER BY o.product DESC LIMIT 2" + it "rewrites Filter id = k into Lookup" $ do + fmap planRoot (optimizedPlan "SELECT name FROM users WHERE id = 2") + `shouldBe` Right (Lookup "users" 2) + it "keeps Lookup result identical to unoptimized" $ do + sameResultAsUnoptimized "SELECT name FROM users WHERE id = 2" - -- ============================================================ - -- Semantic - -- ============================================================ describe "ChuSQL.Semantic" $ do it "rejects an unknown column in WHERE" $ do (parseQuery "SELECT name FROM users WHERE nope > 18" >>= runQuery testDB) @@ -1092,9 +1068,6 @@ main = hspec $ do (parseQuery "SELECT * FROM users u JOIN orders o ON u.id = o.user_id" >>= runQuery testDB) `shouldSatisfy` isRight - -- ============================================================ - -- Op - -- ============================================================ describe "ChuSQL.Algebra.Op" $ do it "renders a plan as indented text" $ do case parseQuery "SELECT name FROM users WHERE age > 18" >>= translate of @@ -1125,12 +1098,203 @@ main = hspec $ do , " Scan Nothing \"users\"" ] + describe "ChuSQL.Storage.IPC" $ do + it "scan returns rows with all three value types after insert" $ do + withTestServer $ \srv -> withServerEnv srv $ do + let row = [("id", VInt 7), ("name", VStr "Alice"), ("flag", VBool True)] + runIPCStorage (insert "users" row) `shouldReturn` Right () + rows <- runIPCStorage (scan "users") + (map sortRow <$> rows) `shouldBe` Right [sortRow row] + + it "scan on a missing table returns Left" $ do + withTestServer $ \srv -> withServerEnv srv $ do + result <- runIPCStorage (scan "no_such_table") + result `shouldSatisfy` isLeft + + it "scan returns all 20 inserted rows in order" $ do + withTestServer $ \srv -> withServerEnv srv $ do + mapM_ (\i -> runIPCStorage (insert "many" [("id", VInt i)])) [1 .. 20 :: Int] + rows <- runIPCStorage (scan "many") + fmap length rows `shouldBe` Right 20 + fmap (sortOn show . map (lookup "id")) rows + `shouldBe` Right (sortOn show (map (Just . VInt) [1 .. 20])) + + it "replaceAll leaves only the new rows" $ do + withTestServer $ \srv -> withServerEnv srv $ do + runIPCStorage (insert "t" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (replaceAll "t" [[("id", VInt 9)], [("id", VInt 8)]]) `shouldReturn` Right () + rows <- runIPCStorage (scan "t") + fmap (sortOn show . map (lookup "id")) rows + `shouldBe` Right (sortOn show [Just (VInt 9), Just (VInt 8)]) + + it "lookupByKey finds a row inserted with an id over the pipe" $ do + withTestServer $ \srv -> withServerEnv srv $ do + let row = [("id", VInt 7), ("name", VStr "Zoe")] + runIPCStorage (insert "keyed" row) `shouldReturn` Right () + found <- runIPCStorage (lookupByKey "keyed" 7) + fmap (fmap sortRow) found `shouldBe` Right (Just (sortRow row)) + missing <- runIPCStorage (lookupByKey "keyed" 8) + missing `shouldBe` Right Nothing + + it "replaceAll rebuilds the index over the pipe" $ do + withTestServer $ \srv -> withServerEnv srv $ do + runIPCStorage (insert "rb" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (replaceAll "rb" [[("id", VInt 9), ("name", VStr "Zoe")]]) `shouldReturn` Right () + old <- runIPCStorage (lookupByKey "rb" 1) + old `shouldBe` Right Nothing + new <- runIPCStorage (lookupByKey "rb" 9) + fmap (fmap sortRow) new + `shouldBe` Right (Just (sortRow [("id", VInt 9), ("name", VStr "Zoe")])) + + it "snapshot lists tables and scans each one to build the database" $ do + withTestServer $ \srv -> withServerEnv srv $ do + runIPCStorage (insert "a" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (insert "b" [("id", VInt 2)]) `shouldReturn` Right () + db <- runIPCStorage snapshot + map fst db `shouldBe` ["a", "b"] + map (length . tableRows . snd) db `shouldBe` [1, 1] + map (map (lookup "id") . tableRows . snd) db `shouldBe` [[Just (VInt 1)], [Just (VInt 2)]] + + it "doListTables returns the tables that exist" $ do + withTestServer $ \srv -> withServerEnv srv $ do + runIPCStorage (insert "t1" [("id", VInt 1)]) `shouldReturn` Right () + runIPCStorage (insert "t2" [("id", VInt 2)]) `shouldReturn` Right () + doListTables `shouldReturn` Right ["t1", "t2"] + + it "rows survive a restart on the same data directory" $ do + located <- locateServer + case located of + Left err -> pendingWith err + Right bin -> do + srv1 <- startServer bin + let dir = dataDir srv1 + withServerEnv srv1 $ runIPCStorage (insert "persist" [("id", VInt 42)]) `shouldReturn` Right () + stopServer srv1 + srv2 <- startServerAt bin dir + rows <- withServerEnv srv2 $ runIPCStorage (scan "persist") + stopServer srv2 + cleanServerDir srv1 + (map sortRow <$> rows) `shouldBe` Right [sortRow [("id", VInt 42)]] + + it "a missing server yields Left without throwing" $ do + withPipeName "chusql-no-such-server" $ do + result <- runIPCStorage (scan "users") + result `shouldSatisfy` isLeft + emptyDB :: Database -emptyDB = [(n, t {tableRows = []}) | (n, t) <- testDB] +emptyDB = [(n, t{tableRows = []}) | (n, t) <- testDB] + +data IPCServer = IPCServer + { pipeName :: String + , dataDir :: FilePath + , processHandle :: ProcessHandle + } + +locateServer :: IO (Either String FilePath) +locateServer = do + mdir <- firstDir [".." "chusql-storage", "chusql-storage"] + case mdir of + Nothing -> pure (Left "chusql-storage directory not found (run these tests inside the repository)") + Just dir -> do + built <- + try (createProcess (proc "cargo" ["build", "--bin", "server"]){cwd = Just dir, std_out = NoStream, std_err = NoStream}) :: + IO (Either IOException (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle)) + case built of + Left e -> pure (Left ("cannot run cargo (Rust is required for the IPC tests): " ++ show e)) + Right (_, _, _, ph) -> do + ec <- waitForProcess ph + case ec of + ExitFailure n -> pure (Left ("cargo build --bin server failed with exit code " ++ show n)) + ExitSuccess -> do + found <- firstExisting [dir "target" "debug" "server.exe", dir "target" "debug" "server"] + pure (maybe (Left "cargo build finished but no server executable was found") Right found) + where + firstDir [] = pure Nothing + firstDir (d : ds) = do + ok <- doesDirectoryExist d + if ok then pure (Just d) else firstDir ds + firstExisting [] = pure Nothing + firstExisting (p : ps) = do + ok <- doesFileExist p + if ok then pure (Just p) else firstExisting ps + +startServer :: FilePath -> IO IPCServer +startServer bin = do + stamp <- show <$> getCPUTime + tmp <- getTemporaryDirectory + let dir = tmp "chusql-hs-IPC" stamp + createDirectoryIfMissing True dir + startServerAt bin dir + +startServerAt :: FilePath -> FilePath -> IO IPCServer +startServerAt bin dir = do + stamp <- show <$> getCPUTime + let name = "chusql-hs-test-" ++ stamp + parentEnv <- getEnvironment + (_, _, _, ph) <- + createProcess + (proc bin []) + { env = Just (("CHUSQL_PIPE", name) : ("CHUSQL_DATA_DIR", dir) : parentEnv) + , std_out = NoStream + , std_err = NoStream + } + let srv = IPCServer name dir ph + waitUntilReady srv 100 + pure srv + +waitUntilReady :: IPCServer -> Int -> IO () +waitUntilReady srv tries = do + probe <- try (withServerEnv srv (sendRequest ReqPing)) :: IO (Either IOException Response) + case probe of + Right RespPong -> pure () + _ | tries > 0 -> threadDelay 50000 >> waitUntilReady srv (tries - 1) + Right _ -> ioError (userError "server is up but ping did not answer pong") + Left e -> ioError e + +stopServer :: IPCServer -> IO () +stopServer srv = do + _ <- try (terminateProcess (processHandle srv)) :: IO (Either IOException ()) + _ <- try (waitForProcess (processHandle srv)) :: IO (Either IOException ExitCode) + pure () + +cleanServerDir :: IPCServer -> IO () +cleanServerDir srv = do + _ <- try (removePathForcibly (dataDir srv)) :: IO (Either IOException ()) + pure () + +withTestServer :: (IPCServer -> IO ()) -> IO () +withTestServer act = do + located <- locateServer + case located of + Left err + | "cannot run cargo" `isInfixOf` err -> pendingWith err + | otherwise -> expectationFailure err + Right bin -> bracket (startServer bin) (\s -> stopServer s >> cleanServerDir s) act + +withServerEnv :: IPCServer -> IO a -> IO a +withServerEnv srv act = do + oldPipe <- lookupEnv "CHUSQL_PIPE" + oldDir <- lookupEnv "CHUSQL_DATA_DIR" + setEnv "CHUSQL_PIPE" (pipeName srv) + setEnv "CHUSQL_DATA_DIR" (dataDir srv) + r <- act + restore "CHUSQL_PIPE" oldPipe + restore "CHUSQL_DATA_DIR" oldDir + pure r + where + restore k = maybe (unsetEnv k) (setEnv k) + +withPipeName :: String -> IO a -> IO a +withPipeName name act = do + old <- lookupEnv "CHUSQL_PIPE" + setEnv "CHUSQL_PIPE" name + r <- act + maybe (unsetEnv "CHUSQL_PIPE") (setEnv "CHUSQL_PIPE") old + pure r + +sortRow :: Row -> Row +sortRow = sortOn fst --- ============================================================ --- Helpers --- ============================================================ isLeft :: Either a b -> Bool isLeft (Left _) = True isLeft (Right _) = False @@ -1167,6 +1331,7 @@ firstFilterCond (Join l r _) = case firstFilterCond l of Just p -> Just p Nothing -> firstFilterCond r firstFilterCond (Scan _ _) = Nothing +firstFilterCond (Lookup _ _) = Nothing anyFilter :: RelOp -> Bool anyFilter (Filter _ _) = True @@ -1175,6 +1340,7 @@ anyFilter (Sort _ x) = anyFilter x anyFilter (Limit _ x) = anyFilter x anyFilter (Join l r _) = anyFilter l || anyFilter r anyFilter (Scan _ _) = False +anyFilter (Lookup _ _) = False stripProjects :: RelOp -> RelOp stripProjects (Project _ x) = stripProjects x @@ -1183,6 +1349,7 @@ stripProjects (Sort spec x) = Sort spec (stripProjects x) stripProjects (Limit n x) = Limit n (stripProjects x) stripProjects (Join l r c) = Join (stripProjects l) (stripProjects r) c stripProjects (Scan a t) = Scan a t +stripProjects (Lookup t k) = Lookup t k projectedPlan :: String -> Either String RelOp projectedPlan sql = do @@ -1205,3 +1372,7 @@ countFilters _ = 0 leftSideFilters :: RelOp -> Maybe Int leftSideFilters (Project _ (Join l _ _)) = Just (countFilters l) leftSideFilters _ = Nothing + +planRoot :: RelOp -> RelOp +planRoot (Project _ x) = planRoot x +planRoot x = x diff --git a/chusql-storage/.gitkeep b/chusql-storage/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/chusql-storage/Cargo.lock b/chusql-storage/Cargo.lock new file mode 100644 index 0000000..f1473e8 --- /dev/null +++ b/chusql-storage/Cargo.lock @@ -0,0 +1,326 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chusql-storage" +version = "0.1.0" +dependencies = [ + "interprocess", + "serde", + "serde_json", + "tempfile", + "toml", +] + +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "interprocess" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb02ffbf25e8ef0f4a95bff054eef75f6d1bf16919474cf2caa63b21997478a" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/chusql-storage/Cargo.toml b/chusql-storage/Cargo.toml new file mode 100644 index 0000000..9a2db8e --- /dev/null +++ b/chusql-storage/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "chusql-storage" +version = "0.1.0" +edition = "2024" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +interprocess = "2" +toml = "0.8" + +[dev-dependencies] +tempfile = "3" diff --git a/chusql-storage/chusql-storage.toml.example b/chusql-storage/chusql-storage.toml.example new file mode 100644 index 0000000..63c2879 --- /dev/null +++ b/chusql-storage/chusql-storage.toml.example @@ -0,0 +1,40 @@ +# ChuSQL 存储服务配置示例 +# +# 用法：把本文件复制成 chusql-storage.toml（放在启动 server 时的当前目录下）， +# 或者让环境变量 CHUSQL_CONFIG 指向任意路径的这个文件。 +# +# 每一项都可以删掉：没写的项回退到内置默认值（下面注释里的值就是默认值）。 +# 优先级：环境变量 > 本文件 > 内置默认值。 +# 启动时 server 会把"读了哪个文件、每个选项从哪来"打进日志（[INFO] 开头那几行）。 + +[page] +# 页大小（字节）：512 ~ 65536 之间的 2 的幂。 +# 注意：改它会让已有数据文件（.db）读不出来（文件长度对不上页大小会直接报错）。 +# 环境变量：CHUSQL_PAGE_SIZE +size = 4096 + +[btree] +# B+ 树一个节点最多几个孩子（>= 3，且必须能放进一页：12 + (order-1)*8 + order*8 <= size）。 +# 注意：改它会让已有索引文件（.idx）对不上文件头里记的值，打开时报错。 +# 环境变量：CHUSQL_BTREE_ORDER +order = 4 + +[storage] +# 数据目录：一张表一个 <表名>.db，一个索引一个 <表名>.idx。 +# 环境变量：CHUSQL_DATA_DIR +data_dir = "data" + +[server] +# 命名管道名（Windows 上实际是 \\.\pipe\<这个名字>）。 +# 环境变量：CHUSQL_PIPE +pipe_name = "chusql-storage" + +[log] +# 日志等级：off / error / warn / info / debug。debug 会多打每个请求和连接。 +# 环境变量：CHUSQL_LOG +level = "info" + +# 颜色不用写在文件里，用环境变量控制：CHUSQL_LOG_COLOR = auto（默认，stderr 是终端才着色） +# / always / never +# 行格式：[等级 本地日期时间] [来源] 消息；来源 core=服务自身、pipe=由管道请求触发 +# ERROR 加粗红、WARN 加粗橙、DEBUG 加粗不变色、INFO 不加样式 diff --git a/chusql-storage/src/bin/server.rs b/chusql-storage/src/bin/server.rs new file mode 100644 index 0000000..ab78164 --- /dev/null +++ b/chusql-storage/src/bin/server.rs @@ -0,0 +1,412 @@ +//! 命名管道服务：按行读写 JSON。Windows 上管道名映射为 \\.\pipe\。 +//! 启动时读配置（环境变量 → TOML → 默认值）并把每项来源打进日志。 + +use std::collections::BTreeSet; +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::sync::Arc; +use std::thread; + +use chusql_storage::catalog::Catalog; +use chusql_storage::config::{self, Config, Loaded}; +use chusql_storage::heap::HeapTable; +use chusql_storage::log; +use chusql_storage::protocol::{Request, Response}; +use chusql_storage::wal::{Wal, WalOp}; +use chusql_storage::{log_debug, log_error, log_info, log_warn}; +use interprocess::local_socket::{ + prelude::*, + GenericNamespaced, + ListenerOptions, +}; +use interprocess::TryClone; + +// * 路径 + +/// 表名 -> 数据文件路径。 +fn table_path(cfg: &Config, table: &str) -> PathBuf { + cfg.data_dir.join(format!("{}.db", table)) +} + +/// 表名 -> 索引文件路径。 +fn index_path(cfg: &Config, table: &str) -> PathBuf { + cfg.data_dir.join(format!("{}.idx", table)) +} + +/// 数据字典文件路径。 +fn catalog_path(cfg: &Config) -> PathBuf { + cfg.data_dir.join("catalog.json") +} + +/// WAL 文件路径。 +fn wal_path(cfg: &Config) -> PathBuf { + cfg.data_dir.join("wal.log") +} + +// * 表操作 + +/// 打开一张表，执行闭包；表不存在就创建。 +fn with_table( + cfg: &Config, + table: &str, + f: impl FnOnce(&mut HeapTable) -> std::io::Result, +) -> std::io::Result { + std::fs::create_dir_all(&cfg.data_dir)?; + let mut t = HeapTable::open(table_path(cfg, table), cfg.page_size)?; + f(&mut t) +} + +/// 打开带索引的表，执行闭包。 +fn with_indexed_table( + cfg: &Config, + table: &str, + f: impl FnOnce(&mut HeapTable) -> std::io::Result, +) -> std::io::Result { + std::fs::create_dir_all(&cfg.data_dir)?; + let mut t = HeapTable::open_indexed( + table_path(cfg, table), + index_path(cfg, table), + cfg.page_size, + cfg.btree_order, + )?; + f(&mut t) +} + +/// 表文件是否存在且非空。 +fn table_exists(cfg: &Config, table: &str) -> bool { + let p = table_path(cfg, table); + std::fs::metadata(&p).map(|m| m.len() > 0).unwrap_or(false) +} + +/// 打开已存在的表；不存在返回 NotFound。 +fn with_existing_table( + cfg: &Config, + table: &str, + f: impl FnOnce(&mut HeapTable) -> std::io::Result, +) -> std::io::Result { + if !table_exists(cfg, table) { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown table: {}", table), + )); + } + let mut t = HeapTable::open(table_path(cfg, table), cfg.page_size)?; + f(&mut t) +} + +// * 数据字典操作 + +/// 读 catalog，按一行数据补列，再写回。 +fn catalog_add_row( + cfg: &Config, + table: &str, + row: &serde_json::Map, +) -> std::io::Result<()> { + let mut c = Catalog::load(catalog_path(cfg))?; + let row: chusql_storage::protocol::Row = row.clone().into_iter().collect(); + c.ensure_columns(table, &row); + c.save(catalog_path(cfg)) +} + +/// 读 catalog，按多行数据补列，再写回。 +fn catalog_add_rows( + cfg: &Config, + table: &str, + rows: &[serde_json::Map], +) -> std::io::Result<()> { + let mut c = Catalog::load(catalog_path(cfg))?; + for r in rows { + let row: chusql_storage::protocol::Row = r.clone().into_iter().collect(); + c.ensure_columns(table, &row); + } + c.save(catalog_path(cfg)) +} + +// * WAL 写路径与恢复 + +/// 把一条 WAL 操作真正落到磁盘（堆表 + 索引 + catalog）。 +fn apply_op(cfg: &Config, op: &WalOp) -> std::io::Result<()> { + match op { + WalOp::Insert { table, row, key } => { + match key { + None => with_table(cfg, table, |t| t.insert(row))?, + Some(k) => with_indexed_table(cfg, table, |t| t.insert_keyed(row, *k))?, + } + let map: serde_json::Map<_, _> = row.clone().into_iter().collect(); + catalog_add_row(cfg, table, &map)?; + Ok(()) + } + WalOp::ReplaceAll { table, rows } => { + with_indexed_table(cfg, table, |t| t.replace_all(rows))?; + let maps: Vec<_> = rows.iter().map(|r| r.clone().into_iter().collect()).collect(); + catalog_add_rows(cfg, table, &maps)?; + Ok(()) + } + } +} + +/// 写前日志 + 写数据 + 清空 WAL。 +/// +/// 顺序：clear（清掉上次已完成的） → append+fsync → apply → clear。 +/// 若中途进程被杀，重启时 WAL 里能读到那条操作并重放。 +fn write_via_wal(cfg: &Config, op: &WalOp) -> std::io::Result<()> { + let wal = Wal::new(wal_path(cfg)); + + // 清掉上一次遗留（正常情况下已经是空的） + wal.clear()?; + + // 先写 WAL 并 fsync + wal.append(op)?; + + // 再写数据 + let result = apply_op(cfg, op); + + // 无论 apply 成功还是返回 Err，都清空 WAL： + // 失败的操作不应被重放；若中途被杀，这里跑不到，WAL 会留到重启时重放。 + wal.clear()?; + + result +} + +/// 启动时重放 WAL 里的残留操作。 +fn recover_wal(cfg: &Config) -> std::io::Result<()> { + let wal = Wal::new(wal_path(cfg)); + match wal.read()? { + None => Ok(()), + Some(op) => { + log_warn!(core, "replaying incomplete WAL entry"); + apply_op(cfg, &op)?; + wal.clear()?; + log_info!(core, "WAL replay complete"); + Ok(()) + } + } +} + +// * 请求分发 + +/// 把一条请求翻译成一条响应。 +fn handle_request(cfg: &Config, req: Request) -> Response { + match req { + Request::Ping => { + log_debug!(pipe, "ping"); + Response::Pong + } + + Request::Scan { table } => { + log_debug!(pipe, "scan table={}", table); + match with_existing_table(cfg, &table, |t| t.scan()) { + Ok(rows) => Response::Rows { rows }, + Err(e) => { + log_warn!(pipe, "scan {}: {}", table, e); + Response::Error { + message: format!("scan {}: {}", table, e), + } + } + } + } + + Request::Insert { table, row, key } => { + log_debug!(pipe, "insert table={} key={:?}", table, key); + let tname = table.clone(); + let op = WalOp::Insert { table, row, key }; + match write_via_wal(cfg, &op) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "insert {}: {}", tname, e); + Response::Error { + message: format!("insert {}: {}", tname, e), + } + } + } + } + + Request::LookupByIndex { table, key } => { + log_debug!(pipe, "lookup_by_index table={} key={}", table, key); + match with_indexed_table(cfg, &table, |t| t.get_by_key(key)) { + Ok(Some(row)) => Response::Rows { rows: vec![row] }, + Ok(None) => Response::Rows { rows: vec![] }, + Err(e) => { + log_warn!(pipe, "lookup_by_index {}: {}", table, e); + Response::Error { + message: format!("lookup_by_index {}: {}", table, e), + } + } + } + } + + Request::ReplaceAll { table, rows } => { + log_debug!(pipe, "replace_all table={} rows={}", table, rows.len()); + let tname = table.clone(); + let op = WalOp::ReplaceAll { table, rows }; + match write_via_wal(cfg, &op) { + Ok(()) => Response::Ok, + Err(e) => { + log_warn!(pipe, "replace_all {}: {}", tname, e); + Response::Error { + message: format!("replace_all {}: {}", tname, e), + } + } + } + } + + Request::ListTables => { + log_debug!(pipe, "list_tables"); + match list_tables(cfg) { + Ok(tables) => Response::Tables { tables }, + Err(e) => { + log_warn!(pipe, "list_tables: {}", e); + Response::Error { + message: format!("list_tables: {}", e), + } + } + } + } + + Request::DescribeTable { table } => { + log_debug!(pipe, "describe_table table={}", table); + match describe_table(cfg, &table) { + Ok(cols) => Response::Schema { columns: cols }, + Err(e) => { + log_warn!(pipe, "describe_table {}: {}", table, e); + Response::Error { + message: format!("describe_table {}: {}", table, e), + } + } + } + } + } +} + +/// 列出表名：catalog 记录 + data 目录下的 .db 文件，取并集。 +fn list_tables(cfg: &Config) -> std::io::Result> { + let mut names = BTreeSet::new(); + + let c = Catalog::load(catalog_path(cfg))?; + for t in c.table_names() { + names.insert(t); + } + + let dir = &cfg.data_dir; + if dir.exists() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name().to_string_lossy().to_string(); + if let Some(stem) = name.strip_suffix(".db") { + names.insert(stem.to_string()); + } + } + } + + Ok(names.into_iter().collect()) +} + +/// 查一张表的 schema。表在 catalog 里没有，但文件存在，返回空列。 +fn describe_table( + cfg: &Config, + table: &str, +) -> std::io::Result> { + let c = Catalog::load(catalog_path(cfg))?; + if let Some(schema) = c.describe(table) { + return Ok(schema.columns.clone()); + } + if table_exists(cfg, table) { + return Ok(Vec::new()); + } + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("unknown table: {}", table), + )) +} + +// * 连接处理 + +/// 处理一条连接：逐行读 JSON 请求，逐行回 JSON 响应。 +fn handle(cfg: &Config, conn: LocalSocketStream) -> std::io::Result<()> { + let mut writer = conn.try_clone()?; + let reader = BufReader::new(conn); + + for line in reader.lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + + let resp = match serde_json::from_str::(&line) { + Ok(req) => handle_request(cfg, req), + Err(e) => { + log_warn!(pipe, "bad request line: {}", e); + Response::Error { + message: format!("parse error: {}", e), + } + } + }; + + writeln!(writer, "{}", serde_json::to_string(&resp).unwrap())?; + writer.flush()?; + } + log_debug!(pipe, "connection closed"); + Ok(()) +} + +// * 主入口 + +/// 把"配置从哪来"打进日志。 +fn log_config(loaded: &Loaded) { + match &loaded.config_path { + Some(p) => log_info!(core, "config file: {}", p.display()), + None => log_info!(core, "config file: none"), + } + for (name, value, origin) in &loaded.origins { + log_info!(core, " {:<17} {} [{}]", name, value, origin.describe()); + } +} + +fn main() -> std::io::Result<()> { + let loaded = match config::load() { + Ok(l) => l, + Err(e) => { + log_error!(core, "config error: {}", e); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, e)); + } + }; + log::init(); + log::set_level(loaded.config.log_level); + + log_info!(core, "chusql-storage {} starting", env!("CARGO_PKG_VERSION")); + log_config(&loaded); + + let cfg = Arc::new(loaded.config); + + // 启动时先重放 WAL，再做其他事 + if let Err(e) = recover_wal(&cfg) { + log_error!(core, "WAL recovery failed: {}", e); + return Err(e); + } + + let ns_name = cfg + .pipe_name + .as_str() + .to_ns_name::() + .map_err(|e| std::io::Error::other(e.to_string()))?; + let listener = ListenerOptions::new().name(ns_name).create_sync()?; + + log_info!( + core, + "listening on pipe {} (\\\\.\\pipe\\{})", + cfg.pipe_name, + cfg.pipe_name + ); + + for conn in listener.incoming() { + let conn = conn?; + let cfg = Arc::clone(&cfg); + thread::spawn(move || { + log_debug!(pipe, "connection accepted"); + if let Err(e) = handle(&cfg, conn) { + log_error!(pipe, "connection error: {}", e); + } + }); + } + Ok(()) +} diff --git a/chusql-storage/src/btree.rs b/chusql-storage/src/btree.rs new file mode 100644 index 0000000..2ec4503 --- /dev/null +++ b/chusql-storage/src/btree.rs @@ -0,0 +1,470 @@ +use std::io; +use std::path::Path; + +use crate::page::{Page, PageFile, PageId}; + +// 磁盘版 B+ 树：每个节点占一页，page 0 存文件头（魔数 + 页大小 + order + 根页号）。 +// 节点页布局（order = 一个节点最多几个孩子，来自配置）： +// [0] 节点类型：0=叶子，1=内部 +// [1..3] 键数量 u16 +// [3..11] 叶子的 next 页号 u64（内部保留） +// [12..12+K*8] 键：K = order-1 个 i64 +// [后面] 叶子：K 个 u64 值；内部：order 个 u64 孩子页号 + +/// 文件头放在这一页。 +const META_PAGE: PageId = 0; +/// 文件头魔数。 +const META_MAGIC: &[u8; 4] = b"CBTR"; +/// 文件头里各字段的偏移。 +const OFF_META_PAGE_SIZE: usize = 4; +const OFF_META_ORDER: usize = 8; +const OFF_META_ROOT: usize = 10; + +/// 节点类型标记。 +const NODE_LEAF: u8 = 0; +const NODE_INTERNAL: u8 = 1; +/// 布局偏移。 +const OFF_TYPE: usize = 0; +const OFF_COUNT: usize = 1; +const OFF_NEXT: usize = 3; +const OFF_KEYS: usize = 12; + +/// 一个 order 的节点占多少字节（内部节点最坏情况：order-1 个键 + order 个孩子）。 +fn node_bytes(order: usize) -> usize { + OFF_KEYS + max_keys(order) * 8 + order * 8 +} + +/// 一个节点最多几个键。 +fn max_keys(order: usize) -> usize { + order - 1 +} + +/// 键后面那片区域的起点。 +fn off_payload(order: usize) -> usize { + OFF_KEYS + max_keys(order) * 8 +} + +/// 这个 order 放得进 size 字节的页吗。 +pub fn fits_in_page(page_size: usize, order: usize) -> bool { + order >= 2 && node_bytes(order) <= page_size +} + +/// 给定页大小，order 最大能取多少（解 12 + (order-1)*8 + order*8 <= page_size）。 +pub fn max_order(page_size: usize) -> usize { + page_size.saturating_sub(4) / 16 +} + +/// 读一个 u8。 +fn read_u8(p: &Page, off: usize) -> u8 { + p.data[off] +} + +/// 写一个 u8。 +fn write_u8(p: &mut Page, off: usize, v: u8) { + p.data[off] = v; +} + +/// 读一个小端 u16。 +fn read_u16(p: &Page, off: usize) -> u16 { + u16::from_le_bytes([p.data[off], p.data[off + 1]]) +} + +/// 写一个小端 u16。 +fn write_u16(p: &mut Page, off: usize, v: u16) { + p.data[off..off + 2].copy_from_slice(&v.to_le_bytes()); +} + +/// 读一个小端 u32。 +fn read_u32(p: &Page, off: usize) -> u32 { + u32::from_le_bytes([p.data[off], p.data[off + 1], p.data[off + 2], p.data[off + 3]]) +} + +/// 写一个小端 u32。 +fn write_u32(p: &mut Page, off: usize, v: u32) { + p.data[off..off + 4].copy_from_slice(&v.to_le_bytes()); +} + +/// 读一个小端 u64。 +fn read_u64(p: &Page, off: usize) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(&p.data[off..off + 8]); + u64::from_le_bytes(b) +} + +/// 写一个小端 u64。 +fn write_u64(p: &mut Page, off: usize, v: u64) { + p.data[off..off + 8].copy_from_slice(&v.to_le_bytes()); +} + +/// 写文件头。 +fn write_meta(page: &mut Page, page_size: usize, order: usize, root: PageId) { + page.zero(); + page.data[0..4].copy_from_slice(META_MAGIC); + write_u32(page, OFF_META_PAGE_SIZE, page_size as u32); + write_u16(page, OFF_META_ORDER, order as u16); + write_u64(page, OFF_META_ROOT, root); +} + +/// 检查文件头和当前配置是否一致（页大小 / order 改了就会对不上）。 +fn check_meta(page: &Page, page_size: usize, order: usize) -> io::Result<()> { + if page.data[0..4] != *META_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "not a chusql index file (bad magic)", + )); + } + let stored_page = read_u32(page, OFF_META_PAGE_SIZE) as usize; + let stored_order = read_u16(page, OFF_META_ORDER) as usize; + if stored_page != page_size || stored_order != order { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "index file was built with page.size={} btree.order={}, config says page.size={} btree.order={}", + stored_page, stored_order, page_size, order + ), + )); + } + Ok(()) +} + +/// 一个节点在内存里的样子。 +enum NodeData { + Leaf { + keys: Vec, + values: Vec, + next: PageId, + }, + Internal { + keys: Vec, + children: Vec, + }, +} + +/// 从一页里读出节点。 +fn read_node(p: &Page, order: usize) -> NodeData { + let ty = read_u8(p, OFF_TYPE); + let n = read_u16(p, OFF_COUNT) as usize; + let payload = off_payload(order); + if ty == NODE_LEAF { + let next = read_u64(p, OFF_NEXT); + let mut keys = Vec::with_capacity(n); + let mut values = Vec::with_capacity(n); + for i in 0..n { + keys.push(read_u64(p, OFF_KEYS + i * 8) as i64); + values.push(read_u64(p, payload + i * 8)); + } + NodeData::Leaf { keys, values, next } + } else { + let mut keys = Vec::with_capacity(n); + let mut children = Vec::with_capacity(n + 1); + for i in 0..n { + keys.push(read_u64(p, OFF_KEYS + i * 8) as i64); + } + for i in 0..(n + 1) { + children.push(read_u64(p, payload + i * 8)); + } + NodeData::Internal { keys, children } + } +} + +/// 把节点写回一页（先清零，避免残留）。 +fn write_node(p: &mut Page, node: &NodeData, order: usize) { + p.zero(); + let payload = off_payload(order); + match node { + NodeData::Leaf { keys, values, next } => { + write_u8(p, OFF_TYPE, NODE_LEAF); + write_u16(p, OFF_COUNT, keys.len() as u16); + write_u64(p, OFF_NEXT, *next); + for (i, k) in keys.iter().enumerate() { + write_u64(p, OFF_KEYS + i * 8, *k as u64); + } + for (i, v) in values.iter().enumerate() { + write_u64(p, payload + i * 8, *v); + } + } + NodeData::Internal { keys, children } => { + write_u8(p, OFF_TYPE, NODE_INTERNAL); + write_u16(p, OFF_COUNT, keys.len() as u16); + for (i, k) in keys.iter().enumerate() { + write_u64(p, OFF_KEYS + i * 8, *k as u64); + } + for (i, c) in children.iter().enumerate() { + write_u64(p, payload + i * 8, *c); + } + } + } +} + +/// 二分找 key 应该下探到的孩子下标。 +fn child_index(keys: &[i64], key: i64) -> usize { + let mut lo = 0; + let mut hi = keys.len(); + while lo < hi { + let mid = (lo + hi) / 2; + if keys[mid] <= key { + lo = mid + 1; + } else { + hi = mid; + } + } + lo +} + +/// 磁盘版 B+ 树。 +pub struct DiskBTree { + file: PageFile, + order: usize, +} + +impl DiskBTree { + /// 打开或新建索引文件；页大小 / order 和文件头里记的不一致就报错。 + pub fn open>(path: P, page_size: usize, order: usize) -> io::Result { + if !fits_in_page(page_size, order) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "btree order {} does not fit in a {}-byte page (max {})", + order, + page_size, + max_order(page_size) + ), + )); + } + let mut file = PageFile::open(path, page_size)?; + if file.num_pages()? == 0 { + let mut meta = Page::new(META_PAGE, page_size); + write_meta(&mut meta, page_size, order, 0); + file.write_page(&meta)?; + } else { + let meta = file.read_page(META_PAGE)?; + check_meta(&meta, page_size, order)?; + } + Ok(DiskBTree { file, order }) + } + + /// 读根页号；0 表示空树。 + fn root(&mut self) -> io::Result> { + let meta = self.file.read_page(META_PAGE)?; + let r = read_u64(&meta, OFF_META_ROOT); + Ok(if r == 0 { None } else { Some(r) }) + } + + /// 写根页号。 + fn set_root(&mut self, r: Option) -> io::Result<()> { + let mut meta = self.file.read_page(META_PAGE)?; + write_u64(&mut meta, OFF_META_ROOT, r.unwrap_or(0)); + self.file.write_page(&meta) + } + + /// 分配一个新页。 + fn alloc(&mut self) -> io::Result { + Ok(self.file.append_page()?.id) + } + + /// 清空整棵树：截断文件后重写文件头（根页号归零）。 + pub fn clear(&mut self) -> io::Result<()> { + let page_size = self.file.page_size(); + self.file.truncate()?; + let mut meta = Page::new(META_PAGE, page_size); + write_meta(&mut meta, page_size, self.order, 0); + self.file.write_page(&meta)?; + Ok(()) + } + + /// 空页，用来拼一个新节点。 + fn new_page(&self, id: PageId) -> Page { + Page::new(id, self.file.page_size()) + } + + /// 按 key 找 value。 + pub fn get(&mut self, key: i64) -> io::Result> { + match self.root()? { + None => Ok(None), + Some(r) => self.get_rec(r, key), + } + } + + /// 递归查找。 + fn get_rec(&mut self, page_id: PageId, key: i64) -> io::Result> { + let p = self.file.read_page(page_id)?; + match read_node(&p, self.order) { + NodeData::Leaf { keys, values, .. } => { + Ok(keys.binary_search(&key).ok().map(|i| values[i])) + } + NodeData::Internal { keys, children } => { + let i = child_index(&keys, key); + self.get_rec(children[i], key) + } + } + } + + /// 插入 / 覆盖一个键值对。 + pub fn insert(&mut self, key: i64, value: u64) -> io::Result<()> { + match self.root()? { + None => { + let id = self.alloc()?; + let node = NodeData::Leaf { + keys: vec![key], + values: vec![value], + next: 0, + }; + let mut p = self.new_page(id); + write_node(&mut p, &node, self.order); + self.file.write_page(&p)?; + self.set_root(Some(id)) + } + Some(r) => { + let (_, split) = self.insert_rec(r, key, value)?; + if let Some((sep, right_id)) = split { + // 根分裂：长高一层。 + let root_id = self.alloc()?; + let node = NodeData::Internal { + keys: vec![sep], + children: vec![r, right_id], + }; + let mut p = self.new_page(root_id); + write_node(&mut p, &node, self.order); + self.file.write_page(&p)?; + self.set_root(Some(root_id)) + } else { + Ok(()) + } + } + } + } + + /// 递归插入；返回可能上提的 (分隔键, 新右页号)。 + fn insert_rec( + &mut self, + page_id: PageId, + key: i64, + value: u64, + ) -> io::Result<(PageId, Option<(i64, PageId)>)> { + let p = self.file.read_page(page_id)?; + let limit = max_keys(self.order); + match read_node(&p, self.order) { + NodeData::Leaf { mut keys, mut values, next } => { + match keys.binary_search(&key) { + Ok(i) => values[i] = value, + Err(i) => { + keys.insert(i, key); + values.insert(i, value); + } + } + if keys.len() <= limit { + let mut p2 = self.new_page(page_id); + write_node(&mut p2, &NodeData::Leaf { keys, values, next }, self.order); + self.file.write_page(&p2)?; + Ok((page_id, None)) + } else { + // 叶子分裂：右叶最小键复制上提。 + let mid = keys.len() / 2; + let rkeys = keys.split_off(mid); + let rvalues = values.split_off(mid); + let sep = rkeys[0]; + + let right_id = self.alloc()?; + let mut rp = self.new_page(right_id); + write_node( + &mut rp, + &NodeData::Leaf { keys: rkeys, values: rvalues, next }, + self.order, + ); + self.file.write_page(&rp)?; + + let mut lp = self.new_page(page_id); + write_node( + &mut lp, + &NodeData::Leaf { keys, values, next: right_id }, + self.order, + ); + self.file.write_page(&lp)?; + + Ok((page_id, Some((sep, right_id)))) + } + } + + NodeData::Internal { mut keys, mut children } => { + let i = child_index(&keys, key); + let (new_child, split) = self.insert_rec(children[i], key, value)?; + children[i] = new_child; + match split { + None => { + let mut p2 = self.new_page(page_id); + write_node(&mut p2, &NodeData::Internal { keys, children }, self.order); + self.file.write_page(&p2)?; + Ok((page_id, None)) + } + Some((sep, right_id)) => { + keys.insert(i, sep); + children.insert(i + 1, right_id); + if keys.len() <= limit { + let mut p2 = self.new_page(page_id); + write_node( + &mut p2, + &NodeData::Internal { keys, children }, + self.order, + ); + self.file.write_page(&p2)?; + Ok((page_id, None)) + } else { + // 内部节点分裂：中间键上提，不再保留原件。 + let mid = keys.len() / 2; + let up = keys[mid]; + let rkeys = keys.split_off(mid + 1); + keys.truncate(mid); + let rchildren = children.split_off(mid + 1); + + let right_id = self.alloc()?; + let mut rp = self.new_page(right_id); + write_node( + &mut rp, + &NodeData::Internal { keys: rkeys, children: rchildren }, + self.order, + ); + self.file.write_page(&rp)?; + + let mut lp = self.new_page(page_id); + write_node( + &mut lp, + &NodeData::Internal { keys, children }, + self.order, + ); + self.file.write_page(&lp)?; + + Ok((page_id, Some((up, right_id)))) + } + } + } + } + } + } + + /// 按 key 升序返回所有键值对。 + pub fn iter_all(&mut self) -> io::Result> { + let mut out = Vec::new(); + if let Some(r) = self.root()? { + self.walk(r, &mut out)?; + } + Ok(out) + } + + /// 中序遍历一棵子树。 + fn walk(&mut self, page_id: PageId, out: &mut Vec<(i64, u64)>) -> io::Result<()> { + let p = self.file.read_page(page_id)?; + match read_node(&p, self.order) { + NodeData::Leaf { keys, values, .. } => { + for i in 0..keys.len() { + out.push((keys[i], values[i])); + } + } + NodeData::Internal { children, .. } => { + for c in children { + self.walk(c, out)?; + } + } + } + Ok(()) + } +} diff --git a/chusql-storage/src/catalog.rs b/chusql-storage/src/catalog.rs new file mode 100644 index 0000000..601e9f8 --- /dev/null +++ b/chusql-storage/src/catalog.rs @@ -0,0 +1,85 @@ +//! 数据字典：记录每张表有哪些列、什么类型。 +//! 存在 data/catalog.json。insert 时自动补列，describe 时读回。 + +use std::collections::BTreeMap; +use std::io; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::protocol::{ColumnType, Row, SchemaColumn}; + +/// 一张表的 schema。 +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct TableSchema { + pub columns: Vec, +} + +/// 整个数据字典。 +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Catalog { + tables: BTreeMap, +} + +impl Catalog { + /// 从文件读；文件不存在返回空字典。 + pub fn load>(path: P) -> io::Result { + if !path.as_ref().exists() { + return Ok(Catalog::default()); + } + let bytes = std::fs::read(path)?; + if bytes.is_empty() { + return Ok(Catalog::default()); + } + serde_json::from_slice(&bytes).map_err(io::Error::other) + } + + /// 写回文件。 + pub fn save>(&self, path: P) -> io::Result<()> { + let bytes = serde_json::to_vec_pretty(self).map_err(io::Error::other)?; + std::fs::write(path, bytes) + } + + /// 一张表的 schema；没有就返回 None。 + pub fn describe(&self, table: &str) -> Option<&TableSchema> { + self.tables.get(table) + } + + /// catalog 里记录的所有表名。 + pub fn table_names(&self) -> Vec { + self.tables.keys().cloned().collect() + } + + /// 按一行数据补列；已有的列不动。 + pub fn ensure_columns(&mut self, table: &str, row: &Row) { + let entry = self.tables.entry(table.to_string()).or_default(); + for (k, v) in row { + if entry.columns.iter().any(|c| &c.name == k) { + continue; + } + if let Some(ty) = infer_type(v) { + entry.columns.push(SchemaColumn { + name: k.clone(), + ty, + }); + } + } + } + + /// 按多行数据补列；replace_all 用。 + pub fn ensure_columns_from_rows(&mut self, table: &str, rows: &[Row]) { + for r in rows { + self.ensure_columns(table, r); + } + } +} + +/// 从 JSON 值推断列类型；不支持的类型返回 None。 +fn infer_type(v: &serde_json::Value) -> Option { + match v { + serde_json::Value::Number(_) => Some(ColumnType::Int), + serde_json::Value::String(_) => Some(ColumnType::Str), + serde_json::Value::Bool(_) => Some(ColumnType::Bool), + _ => None, + } +} diff --git a/chusql-storage/src/config.rs b/chusql-storage/src/config.rs new file mode 100644 index 0000000..2f616e7 --- /dev/null +++ b/chusql-storage/src/config.rs @@ -0,0 +1,281 @@ +use std::path::PathBuf; + +use serde::Deserialize; + +use crate::log::Level; + +// 配置：优先环境变量，其次 TOML 文件，最后内置默认值（下面写的就是默认值）。 +// 每个选项的来源都记在 Loaded::origins 里，server 启动时打进日志——读了哪个文件、 +// 哪些选项走了默认回退，一眼能看出来。 + +/// 默认配置文件名（相对当前工作目录）；可用 CHUSQL_CONFIG 指到别处。 +pub const DEFAULT_CONFIG_PATH: &str = "chusql-storage.toml"; + +/// 默认页大小（字节）。 +pub const DEFAULT_PAGE_SIZE: usize = 4096; +/// 默认 B+ 树节点最大孩子数。 +pub const DEFAULT_BTREE_ORDER: usize = 4; +/// 默认数据目录。 +pub const DEFAULT_DATA_DIR: &str = "data"; +/// 默认管道名。 +pub const DEFAULT_PIPE_NAME: &str = "chusql-storage"; +/// 默认日志等级。 +pub const DEFAULT_LOG_LEVEL: &str = "info"; + +/// 页大小允许的范围。 +const MIN_PAGE_SIZE: usize = 512; +const MAX_PAGE_SIZE: usize = 65536; +/// B+ 树节点至少要放得下 2 个键，分裂规则才成立。 +const MIN_BTREE_ORDER: usize = 3; + +/// 生效的配置。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + pub page_size: usize, + pub btree_order: usize, + pub data_dir: PathBuf, + pub pipe_name: String, + pub log_level: Level, +} + +impl Default for Config { + fn default() -> Self { + Config { + page_size: DEFAULT_PAGE_SIZE, + btree_order: DEFAULT_BTREE_ORDER, + data_dir: PathBuf::from(DEFAULT_DATA_DIR), + pipe_name: DEFAULT_PIPE_NAME.to_string(), + log_level: Level::Info, + } + } +} + +/// 一个选项的来源。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Origin { + Default, + File(PathBuf), + Env(&'static str), +} + +impl Origin { + /// 日志里那一列怎么写。 + pub fn describe(&self) -> String { + match self { + Origin::Default => "default".to_string(), + Origin::File(p) => format!("file {}", p.display()), + Origin::Env(k) => format!("env {}", k), + } + } +} + +/// 生效配置 + 每个选项的来源。 +#[derive(Debug, Clone)] +pub struct Loaded { + pub config: Config, + pub config_path: Option, + pub origins: Vec<(&'static str, String, Origin)>, +} + +/// TOML 文件里的形状：字段全部可选，没写的就当"文件没提这一项"。 +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileConfig { + page: Option, + btree: Option, + storage: Option, + server: Option, + log: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FilePage { + size: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileBtree { + order: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileStorage { + data_dir: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileServer { + pipe_name: Option, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct FileLog { + level: Option, +} + +/// 从真实环境加载：CHUSQL_CONFIG（或当前目录的默认文件）+ 环境变量覆盖。 +pub fn load() -> Result { + let path = match std::env::var("CHUSQL_CONFIG") { + Ok(p) if !p.trim().is_empty() => Some(PathBuf::from(p)), + _ => { + let p = PathBuf::from(DEFAULT_CONFIG_PATH); + if p.is_file() { Some(p) } else { None } + } + }; + let text = match &path { + Some(p) => Some( + std::fs::read_to_string(p) + .map_err(|e| format!("cannot read config file {}: {}", p.display(), e))?, + ), + None => None, + }; + resolve(text.as_deref(), path, &|k| std::env::var(k).ok()) +} + +/// 解析 TOML 文本 + 套环境变量覆盖；测试直接调这个，不碰真实环境。 +pub fn resolve( + text: Option<&str>, + path: Option, + env: &dyn Fn(&str) -> Option, +) -> Result { + let file: FileConfig = match text { + None => FileConfig::default(), + Some(t) => toml::from_str(t).map_err(|e| match &path { + Some(p) => format!("bad config file {}: {}", p.display(), e), + None => format!("bad config: {}", e), + })?, + }; + + let file_origin = Origin::File( + path.clone() + .unwrap_or_else(|| PathBuf::from(DEFAULT_CONFIG_PATH)), + ); + + let (page_size, page_origin) = pick( + "CHUSQL_PAGE_SIZE", + file.page.and_then(|p| p.size), + DEFAULT_PAGE_SIZE, + &file_origin, + env, + |s| { + s.trim() + .parse::() + .map_err(|_| format!("not a number: {}", s)) + }, + )?; + let (btree_order, order_origin) = pick( + "CHUSQL_BTREE_ORDER", + file.btree.and_then(|b| b.order), + DEFAULT_BTREE_ORDER, + &file_origin, + env, + |s| { + s.trim() + .parse::() + .map_err(|_| format!("not a number: {}", s)) + }, + )?; + let (data_dir, dir_origin) = pick( + "CHUSQL_DATA_DIR", + file.storage.and_then(|s| s.data_dir), + DEFAULT_DATA_DIR.to_string(), + &file_origin, + env, + |s| Ok(s.trim().to_string()), + )?; + let (pipe_name, pipe_origin) = pick( + "CHUSQL_PIPE", + file.server.and_then(|s| s.pipe_name), + DEFAULT_PIPE_NAME.to_string(), + &file_origin, + env, + |s| Ok(s.trim().to_string()), + )?; + let (log_level, log_origin) = pick( + "CHUSQL_LOG", + file.log.and_then(|l| l.level), + DEFAULT_LOG_LEVEL.to_string(), + &file_origin, + env, + |s| Ok(s.trim().to_string()), + )?; + let log_level = Level::parse(&log_level) + .ok_or_else(|| format!("log.level is not one of off/error/warn/info/debug: {}", log_level))?; + + if data_dir.is_empty() { + return Err("storage.data_dir must not be empty".to_string()); + } + if pipe_name.is_empty() { + return Err("server.pipe_name must not be empty".to_string()); + } + validate_layout(page_size, btree_order)?; + + let origins = vec![ + ("page.size", page_size.to_string(), page_origin), + ("btree.order", btree_order.to_string(), order_origin), + ("storage.data_dir", data_dir.clone(), dir_origin), + ("server.pipe_name", pipe_name.clone(), pipe_origin), + ("log.level", log_level.name().to_string(), log_origin), + ]; + + Ok(Loaded { + config: Config { + page_size, + btree_order, + data_dir: PathBuf::from(data_dir), + pipe_name, + log_level, + }, + config_path: if text.is_some() { path } else { None }, + origins, + }) +} + +/// 取一个选项：环境变量 → 配置文件 → 默认值，并记下它从哪来。 +fn pick( + env_key: &'static str, + file_value: Option, + default: T, + file_origin: &Origin, + env: &dyn Fn(&str) -> Option, + parse: impl Fn(&str) -> Result, +) -> Result<(T, Origin), String> { + if let Some(raw) = env(env_key) { + let value = parse(&raw).map_err(|e| format!("{}: {}", env_key, e))?; + return Ok((value, Origin::Env(env_key))); + } + if let Some(value) = file_value { + return Ok((value, file_origin.clone())); + } + Ok((default, Origin::Default)) +} + +/// 检查选项本身和选项之间是否说得通。 +fn validate_layout(page_size: usize, btree_order: usize) -> Result<(), String> { + if !(MIN_PAGE_SIZE..=MAX_PAGE_SIZE).contains(&page_size) || !page_size.is_power_of_two() { + return Err(format!( + "page.size must be a power of two between {} and {}, got {}", + MIN_PAGE_SIZE, MAX_PAGE_SIZE, page_size + )); + } + if btree_order < MIN_BTREE_ORDER { + return Err(format!( + "btree.order must be at least {}, got {}", + MIN_BTREE_ORDER, btree_order + )); + } + if !crate::btree::fits_in_page(page_size, btree_order) { + return Err(format!( + "btree.order {} does not fit in a {}-byte page (max {})", + btree_order, + page_size, + crate::btree::max_order(page_size) + )); + } + Ok(()) +} diff --git a/chusql-storage/src/heap.rs b/chusql-storage/src/heap.rs new file mode 100644 index 0000000..1cb87eb --- /dev/null +++ b/chusql-storage/src/heap.rs @@ -0,0 +1,267 @@ +use std::io; +use std::path::Path; + +use crate::btree::DiskBTree; +use crate::page::PageId; +use crate::page::{Page, PageFile}; +use crate::protocol::Row; + +// 堆表：把行按插入顺序塞进页里，支持全表扫描。页大小由配置给（默认 4096）。 +// 页内布局（slotted page）： +// [0..2] slot_count: u16（小端） +// [2..] 槽位目录，每槽 8 字节：offset u32 + len u32 +// [中间] 空闲空间 +// [页尾] 行数据，从后往前长 +// 删除：把槽位的 len 置 0，数据不动（vacuum 以后再说）。 + +/// 页头：slot_count 占 2 字节。 +const HEADER_SIZE: usize = 2; +/// 一个槽位占 8 字节：offset(u32) + len(u32)。 +const SLOT_SIZE: usize = 8; +/// 槽位目录起点。 +const SLOT_DIR_START: usize = HEADER_SIZE; + +/// 从字节里读一个小端 u16。 +fn read_u16(buf: &[u8], off: usize) -> u16 { + u16::from_le_bytes([buf[off], buf[off + 1]]) +} + +/// 往字节里写一个小端 u16。 +fn write_u16(buf: &mut [u8], off: usize, v: u16) { + buf[off..off + 2].copy_from_slice(&v.to_le_bytes()); +} + +/// 从字节里读一个小端 u32。 +fn read_u32(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +/// 往字节里写一个小端 u32。 +fn write_u32(buf: &mut [u8], off: usize, v: u32) { + buf[off..off + 4].copy_from_slice(&v.to_le_bytes()); +} + +/// 一个 (页号, 槽位号) 打包成一个 u64。页号高 48 位，槽位号低 16 位。 +fn pack_position(page_id: PageId, slot: u16) -> u64 { + (page_id << 16) | (slot as u64) +} + +/// 从 u64 解回 (页号, 槽位号)。 +fn unpack_position(v: u64) -> (PageId, u16) { + (v >> 16, (v & 0xFFFF) as u16) +} + +/// 页内操作：不管文件，只管一页里的字节。 +pub struct HeapPage; + +impl HeapPage { + /// 这一页里有多少个槽位。 + pub fn slot_count(page: &Page) -> u16 { + read_u16(&page.data, 0) + } + + /// 改写槽位数量。 + fn set_slot_count(page: &mut Page, v: u16) { + write_u16(&mut page.data, 0, v); + } + + /// 空闲空间下界：槽位目录的末尾。 + fn free_start(page: &Page) -> usize { + SLOT_DIR_START + (Self::slot_count(page) as usize) * SLOT_SIZE + } + + /// 空闲空间上界：所有存活行里最靠前的那个字节偏移（没存活行时就是页尾）。 + fn free_end(page: &Page) -> usize { + let n = Self::slot_count(page); + let mut min_off = page.data.len() as u32; + for i in 0..n as usize { + let slot_off = SLOT_DIR_START + i * SLOT_SIZE; + let off = read_u32(&page.data, slot_off); + let len = read_u32(&page.data, slot_off + 4); + if len > 0 && off < min_off { + min_off = off; + } + } + min_off as usize + } + + /// 试插一行：成功返回槽位号，放不下返回 None。 + pub fn insert_tuple(page: &mut Page, tuple: &[u8]) -> Option { + let n = Self::slot_count(page); + let free_start = Self::free_start(page); + let free_end = Self::free_end(page); + + if free_start + SLOT_SIZE + tuple.len() > free_end { + return None; + } + + // 行数据紧贴已有数据前面 + let tuple_off = free_end - tuple.len(); + page.data[tuple_off..tuple_off + tuple.len()].copy_from_slice(tuple); + + // 追加槽位 + let slot_off = SLOT_DIR_START + (n as usize) * SLOT_SIZE; + write_u32(&mut page.data, slot_off, tuple_off as u32); + write_u32(&mut page.data, slot_off + 4, tuple.len() as u32); + Self::set_slot_count(page, n + 1); + + Some(n) + } + + /// 读槽位里的行字节；槽位不存在或已删除返回 None。 + pub fn get_tuple(page: &Page, slot: u16) -> Option> { + let n = Self::slot_count(page); + if slot >= n { + return None; + } + let slot_off = SLOT_DIR_START + (slot as usize) * SLOT_SIZE; + let off = read_u32(&page.data, slot_off) as usize; + let len = read_u32(&page.data, slot_off + 4) as usize; + if len == 0 { + return None; + } + Some(page.data[off..off + len].to_vec()) + } + + /// 所有槽位号（含已删除的）。 + pub fn iter_slots(page: &Page) -> Vec { + (0..Self::slot_count(page)).collect() + } +} + +/// 堆表：一个文件 = 若干页，顺序追加行。 +pub struct HeapTable { + file: PageFile, + index: Option, +} + +impl HeapTable { + /// 打开（或创建）一张堆表。 + pub fn open>(path: P, page_size: usize) -> io::Result { + Ok(HeapTable { + file: PageFile::open(path, page_size)?, + index: None, + }) + } + + /// 打开带 B+ 树索引的表；索引文件不存在会自动建。 + pub fn open_indexed, Q: AsRef>( + path: P, + index_path: Q, + page_size: usize, + btree_order: usize, + ) -> io::Result { + Ok(HeapTable { + file: PageFile::open(path, page_size)?, + index: Some(DiskBTree::open(index_path, page_size, btree_order)?), + }) + } + + /// 插入一行；不更新索引。 + pub fn insert(&mut self, row: &Row) -> io::Result<()> { + self.insert_returning_position(row).map(|_| ()) + } + + /// 插入一行并返回它的 (页号, 槽位号)。 + pub fn insert_returning_position(&mut self, row: &Row) -> io::Result<(PageId, u16)> { + let tuple = serde_json::to_vec(row).map_err(io::Error::other)?; + + if HEADER_SIZE + SLOT_SIZE + tuple.len() > self.file.page_size() { + return Err(io::Error::other(format!( + "tuple too large: {} bytes", + tuple.len() + ))); + } + + let n = self.file.num_pages()?; + if n > 0 { + let last_id = n - 1; + let mut p = self.file.read_page(last_id)?; + if let Some(slot) = HeapPage::insert_tuple(&mut p, &tuple) { + self.file.write_page(&p)?; + return Ok((last_id, slot)); + } + } + + let mut p = self.file.append_page()?; + let page_id = p.id; + let slot = HeapPage::insert_tuple(&mut p, &tuple).expect("pre-checked fits"); + self.file.write_page(&p)?; + Ok((page_id, slot)) + } + + /// 插入一行，并把 (key, 位置) 写进索引。 + pub fn insert_keyed(&mut self, row: &Row, key: i64) -> io::Result<()> { + let (page_id, slot) = self.insert_returning_position(row)?; + if let Some(idx) = &mut self.index { + idx.insert(key, pack_position(page_id, slot))?; + } + Ok(()) + } + + /// 按索引键取一行；没索引或键不存在返回 None。 + pub fn get_by_key(&mut self, key: i64) -> io::Result> { + let pos = match &mut self.index { + None => return Ok(None), + Some(idx) => match idx.get(key)? { + None => return Ok(None), + Some(p) => p, + }, + }; + let (page_id, slot) = unpack_position(pos); + self.read_at(page_id, slot) + } + + /// 按 (页号, 槽位号) 读一行。 + pub fn read_at(&mut self, page_id: PageId, slot: u16) -> io::Result> { + let p = self.file.read_page(page_id)?; + match HeapPage::get_tuple(&p, slot) { + None => Ok(None), + Some(bytes) => { + let row: Row = serde_json::from_slice(&bytes).map_err(io::Error::other)?; + Ok(Some(row)) + } + } + } + + /// 全表扫描：按页顺序、按槽位顺序返回所有行。 + pub fn scan(&mut self) -> io::Result> { + let n = self.file.num_pages()?; + let mut rows = Vec::new(); + for i in 0..n { + let p = self.file.read_page(i)?; + for slot in HeapPage::iter_slots(&p) { + if let Some(bytes) = HeapPage::get_tuple(&p, slot) { + let row: Row = serde_json::from_slice(&bytes) + .map_err(io::Error::other)?; + rows.push(row); + } + } + } + Ok(rows) + } + + /// 整表改写：清空堆表和索引，再把新行逐行插回去。 + /// 行里有整数 id 就顺手重建索引 —— 堆表位置变了，旧索引必须作废，否则会按老位置读出别的行。 + pub fn replace_all(&mut self, rows: &[Row]) -> io::Result<()> { + self.file.truncate()?; + if let Some(idx) = &mut self.index { + idx.clear()?; + } + for r in rows { + let (page_id, slot) = self.insert_returning_position(r)?; + if let (Some(idx), Some(key)) = (&mut self.index, row_key(r)) { + idx.insert(key, pack_position(page_id, slot))?; + } + } + Ok(()) + } +} + +/// 一行里的索引键：id 列是整数就用它（和 Haskell 侧"主键列叫 id"的约定一致）。 +fn row_key(row: &Row) -> Option { + match row.get("id") { + Some(serde_json::Value::Number(n)) => n.as_i64(), + _ => None, + } +} diff --git a/chusql-storage/src/lib.rs b/chusql-storage/src/lib.rs new file mode 100644 index 0000000..ef44d7a --- /dev/null +++ b/chusql-storage/src/lib.rs @@ -0,0 +1,10 @@ +// 存储层库入口：对外暴露 page / protocol / heap / btree / config / log（server 与集成测试都从这里引）。 + +pub mod btree; +pub mod config; +pub mod heap; +pub mod log; +pub mod page; +pub mod protocol; +pub mod catalog; +pub mod wal; diff --git a/chusql-storage/src/log.rs b/chusql-storage/src/log.rs new file mode 100644 index 0000000..8ed6fbb --- /dev/null +++ b/chusql-storage/src/log.rs @@ -0,0 +1,387 @@ +use std::fmt; +use std::io::{IsTerminal, Write}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::OnceLock; + +// 极简日志：等级存在一个原子变量里，被关掉的等级只花一次原子读——不求值参数、不做格式化。 +// 行格式：[等级 本地日期时间] [来源] 消息 +// - ERROR 加粗红、WARN 加粗橙、DEBUG 加粗不变色（只在 stderr 是终端时着色，重定向到文件时是纯文本） +// - 来源区分 core（服务自身：启动、配置）和 pipe（由管道请求触发） +// 等级由配置文件的 log.level 或环境变量 CHUSQL_LOG 决定。 + +/// 转义序列收尾。 +const RESET: &str = "\u{1b}[0m"; +/// 加粗红（ERROR）。 +const BOLD_RED: &str = "\u{1b}[1m\u{1b}[31m"; +/// 加粗橙（WARN）：256 色里的 208 号。 +const BOLD_ORANGE: &str = "\u{1b}[1m\u{1b}[38;5;208m"; +/// 只加粗（DEBUG）。 +const BOLD: &str = "\u{1b}[1m"; + +/// 日志等级；Off 表示全关。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Level { + Off, + Error, + Warn, + Info, + Debug, +} + +/// 日志来源：core = 服务自身，pipe = 由管道请求触发。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Channel { + Core, + Pipe, +} + +impl Channel { + /// 日志行里的来源标签。 + fn tag(self) -> &'static str { + match self { + Channel::Core => "core", + Channel::Pipe => "pipe", + } + } +} + +impl Level { + /// 解析等级名（大小写不敏感）。 + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "off" => Some(Level::Off), + "error" => Some(Level::Error), + "warn" | "warning" => Some(Level::Warn), + "info" => Some(Level::Info), + "debug" => Some(Level::Debug), + _ => None, + } + } + + /// 等级名（写日志和写配置说明用）。 + pub fn name(self) -> &'static str { + match self { + Level::Off => "off", + Level::Error => "error", + Level::Warn => "warn", + Level::Info => "info", + Level::Debug => "debug", + } + } + + /// 日志行里的标签，固定 5 字符宽，方便对齐。 + fn tag(self) -> &'static str { + match self { + Level::Error => "ERROR", + Level::Warn => "WARN ", + Level::Info => "INFO ", + _ => "DEBUG", + } + } + + /// 这个等级要不要着色（INFO 不着色）。 + fn color(self) -> &'static str { + match self { + Level::Error => BOLD_RED, + Level::Warn => BOLD_ORANGE, + Level::Debug => BOLD, + _ => "", + } + } +} + +/// 当前等级，默认 info。 +static LEVEL: AtomicU8 = AtomicU8::new(Level::Info as u8); + +/// stderr 是不是终端（决定要不要着色），只问一次。 +static COLOR: OnceLock = OnceLock::new(); + +/// 设置全局等级。 +pub fn set_level(level: Level) { + LEVEL.store(level as u8, Ordering::Relaxed); +} + +/// 这个等级要不要输出——热路径上只走这一步（一次原子读 + 比较）。 +#[inline] +pub fn enabled(level: Level) -> bool { + level as u8 <= LEVEL.load(Ordering::Relaxed) +} + +/// 启动时调一次：Windows 上给 stderr 打开 VT 处理，让 ANSI 颜色能被认识。 +pub fn init() { + enable_vt(); +} + +/// 写出一行；只在 enabled 为真时调用。 +pub fn write(channel: Channel, level: Level, args: fmt::Arguments) { + let msg = args.to_string(); + let line = render(level, channel, color_enabled(), &now(), &msg); + let stderr = std::io::stderr(); + let mut out = stderr.lock(); + let _ = writeln!(out, "{}", line); +} + +/// stderr 是终端才着色；CHUSQL_LOG_COLOR=always / never 可以强制开关。 +fn color_enabled() -> bool { + *COLOR.get_or_init(|| match std::env::var("CHUSQL_LOG_COLOR").as_deref().map(str::trim) { + Ok("always") | Ok("yes") | Ok("1") => true, + Ok("never") | Ok("no") | Ok("0") => false, + _ => std::io::stderr().is_terminal(), + }) +} + +/// 拼一行：[等级 时间] [来源] 消息。纯函数，方便测试。 +fn render(level: Level, channel: Channel, color: bool, stamp: &Stamp, msg: &str) -> String { + let tag = level.tag(); + let paint = level.color(); + let level_text = if color && !paint.is_empty() { + format!("{}{}{}", paint, tag, RESET) + } else { + tag.to_string() + }; + format!("[{} {}] [{}] {}", level_text, stamp, channel.tag(), msg) +} + +/// 一个本地日期时间。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Stamp { + pub year: u16, + pub month: u16, + pub day: u16, + pub hour: u16, + pub minute: u16, + pub second: u16, + pub milli: u16, +} + +impl fmt::Display for Stamp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}", + self.year, self.month, self.day, self.hour, self.minute, self.second, self.milli + ) + } +} + +/// 当前本地时间：Windows 上走 kernel32::GetLocalTime。 +#[cfg(windows)] +fn now() -> Stamp { + #[repr(C)] + struct SystemTime { + year: u16, + month: u16, + day_of_week: u16, + day: u16, + hour: u16, + minute: u16, + second: u16, + milliseconds: u16, + } + + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetLocalTime(out: *mut SystemTime); + } + + let mut t = SystemTime { + year: 0, + month: 0, + day_of_week: 0, + day: 0, + hour: 0, + minute: 0, + second: 0, + milliseconds: 0, + }; + unsafe { GetLocalTime(&mut t) }; + Stamp { + year: t.year, + month: t.month, + day: t.day, + hour: t.hour, + minute: t.minute, + second: t.second, + milli: t.milliseconds, + } +} + +/// 当前时间：其他平台没有取本地时区的零依赖办法，退回 UTC。 +#[cfg(not(windows))] +fn now() -> Stamp { + let d = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + let secs = d.as_secs(); + let (year, month, day) = civil_from_days((secs / 86_400) as i64); + Stamp { + year: year as u16, + month: month as u16, + day: day as u16, + hour: ((secs % 86_400) / 3600) as u16, + minute: ((secs % 3_600) / 60) as u16, + second: (secs % 60) as u16, + milli: d.subsec_millis() as u16, + } +} + +/// 把"距 1970-01-01 的天数"换算成 (年, 月, 日)：非 Windows 平台算时间用，单元测试也走这里。 +pub fn civil_from_days(days: i64) -> (i64, u16, u16) { + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u16; + let month = if mp < 10 { mp + 3 } else { mp - 9 } as u16; + (y + i64::from(month <= 2), month, day) +} + +/// Windows：打开 stderr 的 VT 处理。 +#[cfg(windows)] +fn enable_vt() { + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetStdHandle(which: u32) -> *mut core::ffi::c_void; + fn GetConsoleMode(handle: *mut core::ffi::c_void, mode: *mut u32) -> i32; + fn SetConsoleMode(handle: *mut core::ffi::c_void, mode: u32) -> i32; + } + + /// STD_ERROR_HANDLE：-12 的无符号写法。 + const STD_ERROR_HANDLE: u32 = 0xFFFF_FFF4; + /// 让控制台认识 ANSI 转义序列。 + const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004; + + unsafe { + let handle = GetStdHandle(STD_ERROR_HANDLE); + let mut mode = 0u32; + if GetConsoleMode(handle, &mut mode) != 0 { + let _ = SetConsoleMode(handle, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); + } + } +} + +/// 非 Windows：不用做任何事。 +#[cfg(not(windows))] +fn enable_vt() {} + +/// 打一条 debug 日志（关掉时只花一次原子读）。 +#[macro_export] +macro_rules! log_debug { + (core, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Debug, $($arg)*) + }; + (pipe, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Debug, $($arg)*) + }; +} + +/// 打一条 info 日志。 +#[macro_export] +macro_rules! log_info { + (core, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Info, $($arg)*) + }; + (pipe, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Info, $($arg)*) + }; +} + +/// 打一条 warn 日志。 +#[macro_export] +macro_rules! log_warn { + (core, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Warn, $($arg)*) + }; + (pipe, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Warn, $($arg)*) + }; +} + +/// 打一条 error 日志。 +#[macro_export] +macro_rules! log_error { + (core, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Core, $crate::log::Level::Error, $($arg)*) + }; + (pipe, $($arg:tt)*) => { + $crate::log_line!($crate::log::Channel::Pipe, $crate::log::Level::Error, $($arg)*) + }; +} + +/// 上面四个宏的公共部分：先看等级，再看要不要真的写。内部用，别直接调。 +#[doc(hidden)] +#[macro_export] +macro_rules! log_line { + ($chan:expr, $lvl:expr, $($arg:tt)*) => { + if $crate::log::enabled($lvl) { + $crate::log::write($chan, $lvl, format_args!($($arg)*)) + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 天数换算日期：几个已知点 + 两次闰年的 2 月 29 日。 + #[test] + fn civil_from_days_known_dates() { + assert_eq!(civil_from_days(0), (1970, 1, 1)); + assert_eq!(civil_from_days(10_957), (2000, 1, 1)); + assert_eq!(civil_from_days(11_016), (2000, 2, 29)); + assert_eq!(civil_from_days(19_723), (2024, 1, 1)); + assert_eq!(civil_from_days(19_782), (2024, 2, 29)); + } + + /// 行格式：等级 + 本地日期时间 + 来源 + 消息，等级该粗的粗、该色的色。 + #[test] + fn render_has_time_channel_and_colors() { + let stamp = Stamp { + year: 2026, + month: 9, + day: 24, + hour: 13, + minute: 45, + second: 12, + milli: 130, + }; + + assert_eq!( + render(Level::Error, Channel::Pipe, true, &stamp, "boom"), + "[\u{1b}[1m\u{1b}[31mERROR\u{1b}[0m 2026-09-24 13:45:12.130] [pipe] boom" + ); + assert_eq!( + render(Level::Warn, Channel::Core, true, &stamp, "careful"), + "[\u{1b}[1m\u{1b}[38;5;208mWARN \u{1b}[0m 2026-09-24 13:45:12.130] [core] careful" + ); + assert_eq!( + render(Level::Debug, Channel::Pipe, true, &stamp, "detail"), + "[\u{1b}[1mDEBUG\u{1b}[0m 2026-09-24 13:45:12.130] [pipe] detail" + ); + assert_eq!( + render(Level::Info, Channel::Core, true, &stamp, "hello"), + "[INFO 2026-09-24 13:45:12.130] [core] hello" + ); + } + + /// 关掉颜色时整行必须是纯文本（重定向到文件时用）。 + #[test] + fn render_without_color_is_plain() { + let stamp = Stamp { + year: 2026, + month: 9, + day: 24, + hour: 1, + minute: 2, + second: 3, + milli: 4, + }; + assert_eq!( + render(Level::Error, Channel::Core, false, &stamp, "boom"), + "[ERROR 2026-09-24 01:02:03.004] [core] boom" + ); + } +} diff --git a/chusql-storage/src/page.rs b/chusql-storage/src/page.rs new file mode 100644 index 0000000..1e1514b --- /dev/null +++ b/chusql-storage/src/page.rs @@ -0,0 +1,120 @@ +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::Path; + +// 页管理：数据库文件按固定大小的页组织，页大小由配置给（默认 4096，见 config.rs）。 +// 只负责"按页号读 / 写"，不理解页里的内容 —— 内容怎么解释由上层（堆表 / B+ 树 / WAL）决定。 +// 页号从 0 开始，第 n 页在文件中的偏移是 n * page_size。 + +/// 页号：从 0 开始。u64 足够大，不用担心中途用完。 +pub type PageId = u64; + +/// 一页：固定 page_size 字节，外加自己的页号。 +#[derive(Debug, Clone)] +pub struct Page { + pub id: PageId, + pub data: Vec, +} + +impl Page { + /// 新建一页，内容全 0。 + pub fn new(id: PageId, size: usize) -> Self { + Page { + id, + data: vec![0u8; size], + } + } + + /// 清零。复用时先清空，避免残留旧数据。 + pub fn zero(&mut self) { + self.data.fill(0); + } +} + +/// 页文件：负责把 Page 落到磁盘上（不做缓存，每次读写都直接 syscall；缓冲池是后面的阶段）。 +#[derive(Debug)] +pub struct PageFile { + file: File, + page_size: usize, +} + +impl PageFile { + /// 打开（或创建）一个页文件；页大小由配置传进来。 + pub fn open>(path: P, page_size: usize) -> io::Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path)?; + Ok(PageFile { file, page_size }) + } + + /// 这个文件的页大小。 + pub fn page_size(&self) -> usize { + self.page_size + } + + /// 读取指定页号的页；页号超出文件末尾时返回 UnexpectedEof。 + pub fn read_page(&mut self, id: PageId) -> io::Result { + let mut data = vec![0u8; self.page_size]; + self.file.seek(SeekFrom::Start(id * self.page_size as u64))?; + self.file.read_exact(&mut data)?; + Ok(Page { id, data }) + } + + /// 写入一页。若页号超出当前文件末尾，文件会自动扩展。 + pub fn write_page(&mut self, page: &Page) -> io::Result<()> { + if page.data.len() != self.page_size { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "page has {} bytes, this file uses {}", + page.data.len(), + self.page_size + ), + )); + } + self.file + .seek(SeekFrom::Start(page.id * self.page_size as u64))?; + self.file.write_all(&page.data)?; + self.file.flush()?; + Ok(()) + } + + /// 在文件末尾追加一页，返回新页的页号。 + pub fn append_page(&mut self) -> io::Result { + let id = self.num_pages()?; + let page = Page::new(id, self.page_size); + self.write_page(&page)?; + Ok(page) + } + + /// 当前文件里有多少页；长度不是页大小整数倍，说明页大小和文件对不上。 + pub fn num_pages(&mut self) -> io::Result { + let len = self.file.metadata()?.len(); + let size = self.page_size as u64; + if len % size != 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "file length {} is not a multiple of page size {} (did page.size change?)", + len, size + ), + )); + } + Ok(len / size) + } + + /// 强制把文件缓冲刷到磁盘。写入关键路径时用。 + pub fn sync(&mut self) -> io::Result<()> { + self.file.sync_all() + } + + /// 清空文件，游标归零。 + pub fn truncate(&mut self) -> io::Result<()> { + self.file.set_len(0)?; + self.file.seek(SeekFrom::Start(0))?; + Ok(()) + } +} diff --git a/chusql-storage/src/protocol.rs b/chusql-storage/src/protocol.rs new file mode 100644 index 0000000..220b8ca --- /dev/null +++ b/chusql-storage/src/protocol.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +// 协议定义：Haskell ↔ Rust 之间那条"行分隔 JSON"消息的类型（请求 / 响应）。 + +/// 一行：列名到 JSON 值的映射。 +pub type Row = HashMap; + +/// 列类型；跟 Haskell 侧 TInt / TStr / TBool 对应。 +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ColumnType { + Int, + Str, + Bool, +} + +/// 一列的 schema。 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SchemaColumn { + pub name: String, + pub ty: ColumnType, +} + +/// 请求；method 字段做 tag。 +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum Request { + Ping, + Scan { table: String }, + Insert { + table: String, + row: Row, + #[serde(default)] + key: Option, + }, + LookupByIndex { table: String, key: i64 }, + ReplaceAll { table: String, rows: Vec }, + ListTables, + DescribeTable { table: String }, +} + +/// 响应；status 字段做 tag。 +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "lowercase")] +pub enum Response { + Pong, + Rows { rows: Vec }, + Tables { tables: Vec }, + Schema { columns: Vec }, + Ok, + Error { message: String }, +} diff --git a/chusql-storage/src/wal.rs b/chusql-storage/src/wal.rs new file mode 100644 index 0000000..7dbb141 --- /dev/null +++ b/chusql-storage/src/wal.rs @@ -0,0 +1,179 @@ +//! 简化版 WAL：写前日志 + 崩溃恢复。 +//! +//! 条目格式： +//! [u32 body_len][body] +//! body = [u64 LSN][u8 操作类型][u16 表名长度][表名字节][JSON 载荷] +//! +//! 简化版约束： +//! - 最多只有一条未完成的操作。写成功就清空 WAL。 +//! - 不做 checkpoint / redo / undo / 事务。 +//! +//! 崩溃恢复： +//! - 启动时若 WAL 非空，重放那一条，然后清空。 + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::protocol::Row; + +/// 操作类型标记。 +const OP_INSERT: u8 = 1; +const OP_REPLACE_ALL: u8 = 2; + +/// 一条 WAL 操作。 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum WalOp { + Insert { + table: String, + row: Row, + key: Option, + }, + ReplaceAll { + table: String, + rows: Vec, + }, +} + +/// WAL 文件句柄。 +pub struct Wal { + path: PathBuf, +} + +impl Wal { + /// 绑定到 WAL 文件路径；首次 append 时自动创建文件。 + pub fn new>(path: P) -> Self { + Wal { + path: path.as_ref().to_path_buf(), + } + } + + /// 日志文件路径。 + pub fn path(&self) -> &Path { + &self.path + } + + /// 清空 WAL；文件不存在则什么都不做。 + pub fn clear(&self) -> io::Result<()> { + if self.path.exists() { + let f = OpenOptions::new().write(true).open(&self.path)?; + f.set_len(0)?; + f.sync_all()?; + } + Ok(()) + } + + /// 追加一条操作并 fsync。 + pub fn append(&self, op: &WalOp) -> io::Result<()> { + let bytes = encode(op)?; + let mut f = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + f.write_all(&bytes)?; + f.sync_all()?; + Ok(()) + } + + /// 读取当前 WAL；空文件 / 不存在返回 None。 + pub fn read(&self) -> io::Result> { + if !self.path.exists() { + return Ok(None); + } + let mut f = File::open(&self.path)?; + let mut buf = Vec::new(); + f.read_to_end(&mut buf)?; + if buf.is_empty() { + return Ok(None); + } + match decode(&buf) { + Ok(opt) => Ok(opt), + Err(e) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("corrupt WAL: {}", e), + )), + } + } +} + +/// 把一条操作编成字节。 +fn encode(op: &WalOp) -> io::Result> { + // 先编出 (类型, 表名, 载荷) + let (op_type, table, payload) = match op { + WalOp::Insert { table, row, key } => { + let p = serde_json::to_vec(&(row, key)).map_err(io::Error::other)?; + (OP_INSERT, table.clone(), p) + } + WalOp::ReplaceAll { table, rows } => { + let p = serde_json::to_vec(rows).map_err(io::Error::other)?; + (OP_REPLACE_ALL, table.clone(), p) + } + }; + + let table_bytes = table.as_bytes(); + + // body = [u64 LSN][u8 类型][u16 表名长度][表名][载荷] + // 简化版 LSN 固定为 1；完整版会改成真正递增。 + let lsn: u64 = 1; + let mut body = Vec::with_capacity(8 + 1 + 2 + table_bytes.len() + payload.len()); + body.extend_from_slice(&lsn.to_le_bytes()); + body.push(op_type); + body.extend_from_slice(&(table_bytes.len() as u16).to_le_bytes()); + body.extend_from_slice(table_bytes); + body.extend_from_slice(&payload); + + // 整条 = [u32 body_len][body] + let mut out = Vec::with_capacity(4 + body.len()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&body); + Ok(out) +} + +/// 解析一条 WAL 条目。 +/// +/// - `Ok(Some(op))`：成功解析出操作 +/// - `Ok(None)`：尾部被截断（写入一半就断电），按"没有这条"处理 +/// - `Err(_)`：字节流本身损坏，抛错 +fn decode(buf: &[u8]) -> Result, String> { + if buf.len() < 4 { + return Ok(None); + } + let body_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; + if buf.len() < 4 + body_len { + // 尾部被截断——忽略 + return Ok(None); + } + let body = &buf[4..4 + body_len]; + if body.len() < 8 + 1 + 2 { + return Err("body too short".into()); + } + // LSN 简化版忽略 + let _lsn = u64::from_le_bytes([ + body[0], body[1], body[2], body[3], body[4], body[5], body[6], body[7], + ]); + let op_type = body[8]; + let table_len = u16::from_le_bytes([body[9], body[10]]) as usize; + if body.len() < 11 + table_len { + return Err("table name truncated".into()); + } + let table = String::from_utf8(body[11..11 + table_len].to_vec()) + .map_err(|e| format!("bad utf8 table name: {}", e))?; + let payload = &body[11 + table_len..]; + + let op = match op_type { + OP_INSERT => { + let (row, key): (Row, Option) = serde_json::from_slice(payload) + .map_err(|e| format!("bad insert payload: {}", e))?; + WalOp::Insert { table, row, key } + } + OP_REPLACE_ALL => { + let rows: Vec = serde_json::from_slice(payload) + .map_err(|e| format!("bad replace_all payload: {}", e))?; + WalOp::ReplaceAll { table, rows } + } + other => return Err(format!("unknown op type: {}", other)), + }; + Ok(Some(op)) +} diff --git a/chusql-storage/tests/btree_test.rs b/chusql-storage/tests/btree_test.rs new file mode 100644 index 0000000..998a916 --- /dev/null +++ b/chusql-storage/tests/btree_test.rs @@ -0,0 +1,150 @@ +use chusql_storage::btree::DiskBTree; +use chusql_storage::config::{DEFAULT_BTREE_ORDER, DEFAULT_PAGE_SIZE}; + +// B+ 树测试：空树查询、插入后查找、重开文件数据仍在、覆盖同一个键、升序遍历、 +// 倒序插入 500 个键、页大小 / order 可以配置、配置和文件头对不上时报错。 + +/// 空树的 get 返回 None，遍历返回空。 +#[test] +fn empty_tree_get() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + ) + .unwrap(); + assert_eq!(t.get(1).unwrap(), None); + assert_eq!(t.iter_all().unwrap(), vec![]); +} + +/// 插 100 个键，每个都能按 key 查回来。 +#[test] +fn insert_then_get() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + ) + .unwrap(); + for i in 0..100 { + t.insert(i, i as u64 * 10).unwrap(); + } + for i in 0..100 { + assert_eq!(t.get(i).unwrap(), Some(i as u64 * 10)); + } +} + +/// 关掉再打开同一个索引文件，数据还在。 +#[test] +fn reopen_persists() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.idx"); + { + let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER).unwrap(); + for i in 0..50 { + t.insert(i, i as u64).unwrap(); + } + } + let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER).unwrap(); + for i in 0..50 { + assert_eq!(t.get(i).unwrap(), Some(i as u64)); + } +} + +/// 同一个键插两次，后写的值生效。 +#[test] +fn update_existing() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + ) + .unwrap(); + t.insert(5, 100).unwrap(); + t.insert(5, 200).unwrap(); + assert_eq!(t.get(5).unwrap(), Some(200)); +} + +/// 倒序插入后遍历，结果仍按 key 升序。 +#[test] +fn iter_all_sorted() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + ) + .unwrap(); + for i in (0..50).rev() { + t.insert(i, i as u64).unwrap(); + } + let all = t.iter_all().unwrap(); + assert_eq!(all.len(), 50); + for (i, (k, _)) in all.iter().enumerate() { + assert_eq!(*k, i as i64); + } +} + +/// 倒序插入 500 个键（必然多次分裂），每个键都能查到。 +#[test] +fn reverse_order_insert() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open( + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + DEFAULT_BTREE_ORDER, + ) + .unwrap(); + for i in (0..500).rev() { + t.insert(i, i as u64).unwrap(); + } + for i in 0..500 { + assert_eq!(t.get(i).unwrap(), Some(i as u64)); + } +} + +/// 页大小和 order 都可以配置：512 字节的页 + 8 个孩子照样能插能查。 +#[test] +fn custom_page_size_and_order() { + let dir = tempfile::tempdir().unwrap(); + let mut t = DiskBTree::open(dir.path().join("t.idx"), 512, 8).unwrap(); + for i in 0..200 { + t.insert(i, i as u64 + 1).unwrap(); + } + for i in 0..200 { + assert_eq!(t.get(i).unwrap(), Some(i as u64 + 1)); + } +} + +/// 文件头里记着页大小和 order：换配置再打开要报错，而不是读出乱七八糟的东西。 +#[test] +fn config_mismatch_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.idx"); + { + let mut t = DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER).unwrap(); + t.insert(1, 1).unwrap(); + } + assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, 8).is_err()); + assert!(DiskBTree::open(&path, 8192, DEFAULT_BTREE_ORDER).is_err()); + assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER).is_ok()); +} + +/// 不是本项目的索引文件（比如随便一个空文件）要报错。 +#[test] +fn not_an_index_file_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("junk.idx"); + std::fs::write(&path, vec![7u8; DEFAULT_PAGE_SIZE]).unwrap(); + assert!(DiskBTree::open(&path, DEFAULT_PAGE_SIZE, DEFAULT_BTREE_ORDER).is_err()); +} + +/// order 放不进一页时直接拒绝。 +#[test] +fn order_too_large_for_page_errors() { + let dir = tempfile::tempdir().unwrap(); + assert!(DiskBTree::open(dir.path().join("t.idx"), 512, 300).is_err()); +} diff --git a/chusql-storage/tests/catalog_test.rs b/chusql-storage/tests/catalog_test.rs new file mode 100644 index 0000000..8faa679 --- /dev/null +++ b/chusql-storage/tests/catalog_test.rs @@ -0,0 +1,59 @@ +use chusql_storage::catalog::Catalog; +use chusql_storage::protocol::{ColumnType, Row}; +use serde_json::json; + +fn row(pairs: &[(&str, serde_json::Value)]) -> Row { + let mut m = Row::new(); + for (k, v) in pairs { + m.insert((*k).to_string(), v.clone()); + } + m +} + +#[test] +fn empty_file_gives_empty_catalog() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + let c = Catalog::load(&path).unwrap(); + assert_eq!(c.table_names().len(), 0); +} + +#[test] +fn ensure_then_describe() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + + let mut c = Catalog::load(&path).unwrap(); + c.ensure_columns("users", &row(&[("id", json!(1)), ("name", json!("Alice"))])); + c.save(&path).unwrap(); + + let c2 = Catalog::load(&path).unwrap(); + let s = c2.describe("users").unwrap(); + assert_eq!(s.columns.len(), 2); + let names: Vec<&str> = s.columns.iter().map(|c| c.name.as_str()).collect(); + assert!(names.contains(&"id")); + assert!(names.contains(&"name")); + let id_col = s.columns.iter().find(|c| c.name == "id").unwrap(); + assert_eq!(id_col.ty, ColumnType::Int); + let name_col = s.columns.iter().find(|c| c.name == "name").unwrap(); + assert_eq!(name_col.ty, ColumnType::Str); +} + +#[test] +fn second_insert_does_not_duplicate_columns() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + + let mut c = Catalog::load(&path).unwrap(); + c.ensure_columns("users", &row(&[("id", json!(1))])); + c.ensure_columns("users", &row(&[("id", json!(2))])); + assert_eq!(c.describe("users").unwrap().columns.len(), 1); +} + +#[test] +fn describe_unknown_returns_none() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.json"); + let c = Catalog::load(&path).unwrap(); + assert!(c.describe("nope").is_none()); +} diff --git a/chusql-storage/tests/config_test.rs b/chusql-storage/tests/config_test.rs new file mode 100644 index 0000000..6c30aa9 --- /dev/null +++ b/chusql-storage/tests/config_test.rs @@ -0,0 +1,131 @@ +use std::path::PathBuf; + +use chusql_storage::config::{ + self, Config, DEFAULT_BTREE_ORDER, DEFAULT_DATA_DIR, DEFAULT_PAGE_SIZE, DEFAULT_PIPE_NAME, + Origin, +}; +use chusql_storage::log::Level; + +// 配置测试：没有文件时全默认、文件里写了才覆盖、环境变量优先于文件、 +// 非法值 / 未知键要报错、每个选项的来源都记得下来。 + +/// 没有配置文件、没有环境变量：全部走内置默认值。 +#[test] +fn defaults_without_file() { + let loaded = config::resolve(None, None, &|_| None).unwrap(); + assert_eq!(loaded.config, Config::default()); + assert!(loaded.config_path.is_none()); + assert!(loaded.origins.iter().all(|(_, _, o)| *o == Origin::Default)); + assert_eq!(loaded.config.page_size, DEFAULT_PAGE_SIZE); + assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER); + assert_eq!(loaded.config.data_dir, PathBuf::from(DEFAULT_DATA_DIR)); + assert_eq!(loaded.config.pipe_name, DEFAULT_PIPE_NAME); + assert_eq!(loaded.config.log_level, Level::Info); +} + +/// 文件里只写了一项：这一项生效，其余回退默认（并且来源记得清清楚楚）。 +#[test] +fn file_overrides_only_written_keys() { + let path = PathBuf::from("t.toml"); + let loaded = config::resolve(Some("[page]\nsize = 8192\n"), Some(path.clone()), &|_| None) + .unwrap(); + + assert_eq!(loaded.config.page_size, 8192); + assert_eq!(loaded.config.btree_order, DEFAULT_BTREE_ORDER); + assert_eq!(loaded.config.data_dir, PathBuf::from(DEFAULT_DATA_DIR)); + assert_eq!(loaded.config_path, Some(path.clone())); + + let page = loaded.origins.iter().find(|(n, _, _)| *n == "page.size").unwrap(); + assert_eq!(page.2, Origin::File(path)); + let order = loaded.origins.iter().find(|(n, _, _)| *n == "btree.order").unwrap(); + assert_eq!(order.2, Origin::Default); +} + +/// 五项全写：值都生效，来源都是文件。 +#[test] +fn file_all_keys() { + let text = "[page]\nsize = 8192\n[btree]\norder = 32\n[storage]\ndata_dir = \"mydata\"\n[server]\npipe_name = \"mypipe\"\n[log]\nlevel = \"debug\"\n"; + let path = PathBuf::from("t.toml"); + let loaded = config::resolve(Some(text), Some(path.clone()), &|_| None).unwrap(); + + assert_eq!(loaded.config.page_size, 8192); + assert_eq!(loaded.config.btree_order, 32); + assert_eq!(loaded.config.data_dir, PathBuf::from("mydata")); + assert_eq!(loaded.config.pipe_name, "mypipe"); + assert_eq!(loaded.config.log_level, Level::Debug); + assert!( + loaded + .origins + .iter() + .all(|(_, _, o)| *o == Origin::File(path.clone())) + ); +} + +/// 环境变量优先于文件。 +#[test] +fn env_beats_file() { + let text = "[storage]\ndata_dir = \"from-file\"\n"; + let env = |k: &str| (k == "CHUSQL_DATA_DIR").then(|| "from-env".to_string()); + let loaded = config::resolve(Some(text), Some(PathBuf::from("t.toml")), &env).unwrap(); + + assert_eq!(loaded.config.data_dir, PathBuf::from("from-env")); + let dir = loaded + .origins + .iter() + .find(|(n, _, _)| *n == "storage.data_dir") + .unwrap(); + assert_eq!(dir.2, Origin::Env("CHUSQL_DATA_DIR")); +} + +/// 日志等级写错要报错，而不是悄悄按默认跑。 +#[test] +fn rejects_bad_log_level() { + let err = config::resolve(Some("[log]\nlevel = \"loud\"\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("log.level"), "got {}", err); +} + +/// 页大小必须是 512..65536 之间的 2 的幂。 +#[test] +fn rejects_bad_page_size() { + let err = config::resolve(Some("[page]\nsize = 3000\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("page.size"), "got {}", err); + + let err = config::resolve(Some("[page]\nsize = 128\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("page.size"), "got {}", err); +} + +/// order 太小、或者放不进一页，都要报错。 +#[test] +fn rejects_bad_btree_order() { + let err = config::resolve(Some("[btree]\norder = 2\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("btree.order"), "got {}", err); + + let err = config::resolve(Some("[btree]\norder = 300\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("does not fit"), "got {}", err); +} + +/// 配置里出现认不出来的键要报错（拼错了不能当没看见）。 +#[test] +fn rejects_unknown_key() { + let err = config::resolve(Some("[page]\nsizes = 4096\n"), None, &|_| None).unwrap_err(); + assert!(err.contains("bad config"), "got {}", err); +} + +/// 环境变量给了非法值也要报错。 +#[test] +fn rejects_bad_env_value() { + let env = |k: &str| (k == "CHUSQL_PAGE_SIZE").then(|| "big".to_string()); + let err = config::resolve(None, None, &env).unwrap_err(); + assert!(err.contains("CHUSQL_PAGE_SIZE"), "got {}", err); +} + +/// 来源文案要能说清"哪来的"。 +#[test] +fn origin_description() { + assert_eq!(Origin::Default.describe(), "default"); + assert_eq!(Origin::Env("CHUSQL_PIPE").describe(), "env CHUSQL_PIPE"); + assert_eq!( + Origin::File(PathBuf::from("a.toml")).describe(), + format!("file {}", PathBuf::from("a.toml").display()) + ); +} diff --git a/chusql-storage/tests/heap_test.rs b/chusql-storage/tests/heap_test.rs new file mode 100644 index 0000000..88348d2 --- /dev/null +++ b/chusql-storage/tests/heap_test.rs @@ -0,0 +1,143 @@ +use chusql_storage::config::DEFAULT_PAGE_SIZE; +use chusql_storage::heap::HeapTable; +use chusql_storage::protocol::Row; +use serde_json::json; + +// 堆表测试：插入 + 全表扫描、跨页多行、大行、超大行报错、重开文件后数据还在、自定义页大小。 + +/// 造一行测试数据（列名 → JSON 值）。 +fn row(pairs: &[(&str, serde_json::Value)]) -> Row { + let mut m = Row::new(); + for (k, v) in pairs { + m.insert((*k).to_string(), v.clone()); + } + m +} + +/// 插一行再扫描，能原样读回来。 +#[test] +fn insert_and_scan_one_row() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + t.insert(&row(&[("id", json!(1)), ("name", json!("Alice"))])) + .unwrap(); + + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["id"], json!(1)); + assert_eq!(rows[0]["name"], json!("Alice")); +} + +/// 插 500 行（必然跨多页），扫描回来的行数和顺序都要对。 +#[test] +fn insert_many_rows_cross_pages() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + for i in 0..500 { + t.insert(&row(&[("id", json!(i))])).unwrap(); + } + + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), 500); + // 顺序保持 + for (i, r) in rows.iter().enumerate() { + assert_eq!(r["id"], json!(i as i64)); + } +} + +/// 约 2KB 的大行也要能放进一页。 +#[test] +fn large_row_still_fits() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + let big = "x".repeat(2000); + t.insert(&row(&[("data", json!(big.clone()))])).unwrap(); + + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["data"], json!(big)); +} + +/// 超过一页容量的行要报错，而不是悄悄写坏。 +#[test] +fn row_too_large_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + let huge = "x".repeat(10_000); + let err = t.insert(&row(&[("data", json!(huge))])); + assert!(err.is_err()); +} + +/// 关掉再重新打开同一个文件，之前插入的数据还在。 +#[test] +fn reopen_and_scan() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + + { + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + t.insert(&row(&[("id", json!(1))])).unwrap(); + t.insert(&row(&[("id", json!(2))])).unwrap(); + } + + let mut t = HeapTable::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + let rows = t.scan().unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["id"], json!(1)); + assert_eq!(rows[1]["id"], json!(2)); +} + +/// 页大小可以配置：512 字节的页装得下小行、放不下大行，行数不变。 +#[test] +fn custom_page_size() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("small.db"); + let mut t = HeapTable::open(&path, 512).unwrap(); + + for i in 0..50 { + t.insert(&row(&[("id", json!(i))])).unwrap(); + } + assert_eq!(t.scan().unwrap().len(), 50); + + let big = "x".repeat(600); + assert!(t.insert(&row(&[("data", json!(big))])).is_err()); +} + +/// 整表改写之后索引必须跟着重建：旧键查不到，新行按 id 能查到。 +#[test] +fn replace_all_rebuilds_index() { + let dir = tempfile::tempdir().unwrap(); + let mut t = HeapTable::open_indexed( + dir.path().join("t.db"), + dir.path().join("t.idx"), + DEFAULT_PAGE_SIZE, + chusql_storage::config::DEFAULT_BTREE_ORDER, + ) + .unwrap(); + + t.insert_keyed(&row(&[("id", json!(1)), ("name", json!("Alice"))]), 1) + .unwrap(); + t.insert_keyed(&row(&[("id", json!(2)), ("name", json!("Bob"))]), 2) + .unwrap(); + assert!(t.get_by_key(1).unwrap().is_some()); + + t.replace_all(&[ + row(&[("id", json!(7)), ("name", json!("Zoe"))]), + row(&[("name", json!("NoId"))]), + ]) + .unwrap(); + + assert!(t.get_by_key(1).unwrap().is_none(), "旧键必须作废"); + assert!(t.get_by_key(2).unwrap().is_none()); + let got = t.get_by_key(7).unwrap().expect("新行应该按 id 建好索引"); + assert_eq!(got["name"], json!("Zoe")); + assert_eq!(t.scan().unwrap().len(), 2); +} diff --git a/chusql-storage/tests/page_test.rs b/chusql-storage/tests/page_test.rs new file mode 100644 index 0000000..729ba6e --- /dev/null +++ b/chusql-storage/tests/page_test.rs @@ -0,0 +1,129 @@ +use chusql_storage::config::DEFAULT_PAGE_SIZE; +use chusql_storage::page::{Page, PageFile}; + +// 页文件测试：一页的读写、页号越界报错、追加页让文件变长、多页互不干扰、覆写同一页、 +// 自定义页大小、页大小和文件对不上时报错。 + +/// 写入一页再读回来，内容和长度都应该一致。 +#[test] +fn write_then_read_page() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + let mut p = Page::new(0, DEFAULT_PAGE_SIZE); + p.data[0] = 42; + p.data[DEFAULT_PAGE_SIZE - 1] = 7; + pf.write_page(&p).unwrap(); + + let read = pf.read_page(0).unwrap(); + assert_eq!(read.data[0], 42); + assert_eq!(read.data[DEFAULT_PAGE_SIZE - 1], 7); + assert_eq!(read.data.len(), DEFAULT_PAGE_SIZE); +} + +/// 空文件读第 0 页应该报错。 +#[test] +fn read_out_of_range_returns_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + assert!(pf.read_page(0).is_err()); +} + +/// 每 append 一页，num_pages 就加一。 +#[test] +fn append_grows_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + assert_eq!(pf.num_pages().unwrap(), 0); + + pf.append_page().unwrap(); + assert_eq!(pf.num_pages().unwrap(), 1); + + pf.append_page().unwrap(); + pf.append_page().unwrap(); + assert_eq!(pf.num_pages().unwrap(), 3); +} + +/// 页 0 和页 1 的内容互不影响。 +#[test] +fn two_pages_independent() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + let mut a = Page::new(0, DEFAULT_PAGE_SIZE); + a.data[0] = 1; + let mut b = Page::new(1, DEFAULT_PAGE_SIZE); + b.data[0] = 2; + pf.write_page(&a).unwrap(); + pf.write_page(&b).unwrap(); + + assert_eq!(pf.read_page(0).unwrap().data[0], 1); + assert_eq!(pf.read_page(1).unwrap().data[0], 2); +} + +/// 覆写同一页：读到的是新内容，页数不变。 +#[test] +fn overwrite_page() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + + let mut a = Page::new(0, DEFAULT_PAGE_SIZE); + a.data[0] = 1; + pf.write_page(&a).unwrap(); + + let mut b = Page::new(0, DEFAULT_PAGE_SIZE); + b.data[0] = 99; + pf.write_page(&b).unwrap(); + + assert_eq!(pf.read_page(0).unwrap().data[0], 99); + assert_eq!(pf.num_pages().unwrap(), 1); +} + +/// 页大小可以配置：512 字节的页照样读写，文件按 512 字节对齐。 +#[test] +fn custom_page_size() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("small.db"); + + let mut pf = PageFile::open(&path, 512).unwrap(); + assert_eq!(pf.page_size(), 512); + + let mut p = Page::new(0, 512); + p.data[511] = 9; + pf.write_page(&p).unwrap(); + + assert_eq!(pf.read_page(0).unwrap().data[511], 9); + assert_eq!(pf.num_pages().unwrap(), 1); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 512); +} + +/// 文件长度不是页大小整数倍（比如页大小被改过）要报错，而不是算出错误的页数。 +#[test] +fn mismatched_file_length_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("odd.db"); + std::fs::write(&path, vec![0u8; 100]).unwrap(); + + let mut pf = PageFile::open(&path, DEFAULT_PAGE_SIZE).unwrap(); + assert!(pf.num_pages().is_err()); +} + +/// 页大小对不上的页不许写进去。 +#[test] +fn write_wrong_sized_page_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + + let mut pf = PageFile::open(&path, 512).unwrap(); + assert!(pf.write_page(&Page::new(0, 1024)).is_err()); +} diff --git a/chusql-storage/tests/pipe_test.rs b/chusql-storage/tests/pipe_test.rs new file mode 100644 index 0000000..a65f202 --- /dev/null +++ b/chusql-storage/tests/pipe_test.rs @@ -0,0 +1,284 @@ +use std::io::{BufRead, BufReader, Write}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::Duration; + +use interprocess::local_socket::{ + prelude::*, + GenericNamespaced +}; +use interprocess::TryClone; + +// 命名管道集成测试：起 server 子进程，用唯一管道名，测试之间互不干扰。 + +/// 生成唯一管道名，避免测试之间抢占。 +fn unique_pipe_name() -> String { + let id = std::process::id(); + let ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("chusql-test-{}-{}", id, ns) +} + +/// 持有 server 子进程，Drop 时杀掉。 +struct ServerProc(Child); + +impl Drop for ServerProc { + /// 测试结束时杀掉子进程并回收，避免留下孤儿进程。 + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// 启动 server 子进程，并等管道可连（最多 5 秒）。 +fn start_server(pipe: &str) -> (ServerProc, tempfile::TempDir) { + let data = tempfile::tempdir().unwrap(); + + let child = Command::new(env!("CARGO_BIN_EXE_server")) + .env("CHUSQL_PIPE", pipe) + .env("CHUSQL_DATA_DIR", data.path()) + .env("CHUSQL_PAGE_SIZE", "4096") + .env("CHUSQL_BTREE_ORDER", "4") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to spawn server"); + + let srv = ServerProc(child); + + for _ in 0..100 { + if connect(pipe).is_ok() { + return (srv, data); + } + thread::sleep(Duration::from_millis(50)); + } + panic!("server did not start within 5s"); +} + +/// 连到指定管道。 +fn connect(pipe: &str) -> std::io::Result { + let name = pipe + .to_ns_name::() + .map_err(|e| std::io::Error::other(e.to_string()))?; + LocalSocketStream::connect(name) +} + +/// 发一行请求，读一行响应。 +fn send(stream: &mut LocalSocketStream, line: &str) -> String { + stream.write_all(line.as_bytes()).unwrap(); + stream.write_all(b"\n").unwrap(); + stream.flush().unwrap(); + + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut buf = String::new(); + reader.read_line(&mut buf).unwrap(); + buf.trim().to_string() +} + +/// ping 应该回 pong。 +#[test] +fn ping_pong() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + + let mut c = connect(&pipe).unwrap(); + assert_eq!(send(&mut c, r#"{"method":"ping"}"#), r#"{"status":"pong"}"#); +} + +/// 插一行再 scan，能扫到刚插进去的数据。 +#[test] +fn insert_then_scan() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + let r1 = send( + &mut c, + r#"{"method":"insert","table":"users","row":{"id":1,"name":"Alice"}}"#, + ); + assert!(r1.contains(r#""status":"ok""#), "insert response: {}", r1); + + let r2 = send(&mut c, r#"{"method":"scan","table":"users"}"#); + assert!(r2.contains("Alice"), "scan response: {}", r2); +} + +#[test] +fn scan_unknown_table_errors() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + let r = send(&mut c, r#"{"method":"scan","table":"nope"}"#); + assert!(r.contains(r#""status":"error""#), "got {}", r); + assert!(r.contains("unknown table"), "got {}", r); +} + +#[test] +fn list_tables_returns_created_tables() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + send(&mut c, r#"{"method":"insert","table":"users","row":{"id":1}}"#); + send(&mut c, r#"{"method":"insert","table":"orders","row":{"id":1}}"#); + + let r = send(&mut c, r#"{"method":"list_tables"}"#); + assert!(r.contains("users"), "got {}", r); + assert!(r.contains("orders"), "got {}", r); +} + +#[test] +fn replace_all_replaces_rows() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + send(&mut c, r#"{"method":"insert","table":"users","row":{"id":1,"name":"Alice"}}"#); + send(&mut c, r#"{"method":"insert","table":"users","row":{"id":2,"name":"Bob"}}"#); + + let r = send( + &mut c, + r#"{"method":"replace_all","table":"users","rows":[{"id":9,"name":"Zoe"}]}"#, + ); + assert!(r.contains(r#""status":"ok""#), "replace_all response: {}", r); + + let r = send(&mut c, r#"{"method":"scan","table":"users"}"#); + assert!(r.contains("Zoe"), "got {}", r); + assert!(!r.contains("Alice"), "old row should be gone: {}", r); +} + +#[test] +fn insert_with_key_then_lookup() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + // 带索引键插三行 + for (id, name) in [(1, "Alice"), (2, "Bob"), (3, "Carol")] { + let req = format!( + r#"{{"method":"insert","table":"idx_users","row":{{"id":{},"name":"{}"}},"key":{}}}"#, + id, name, id + ); + let r = send(&mut c, &req); + assert!(r.contains(r#""status":"ok""#), "insert: {}", r); + } + + // 命中 + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"idx_users","key":2}"#); + assert!(r.contains("Bob"), "lookup: {}", r); + + // 未命中 + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"idx_users","key":99}"#); + assert!(r.contains(r#""rows":[]"#), "miss: {}", r); +} + +#[test] +fn insert_without_key_does_not_index() { + let pipe = unique_pipe_name(); + let (_srv, _data) = start_server(&pipe); + let mut c = connect(&pipe).unwrap(); + + send(&mut c, r#"{"method":"insert","table":"nokey","row":{"id":1}}"#); + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"nokey","key":1}"#); + assert!(r.contains(r#""rows":[]"#), "got: {}", r); +} + +#[test] +fn lookup_survives_reopen() { + let pipe = unique_pipe_name(); + let data = tempfile::tempdir().unwrap(); + + // 第一次：写 3 行带 key + { + let exe = env!("CARGO_BIN_EXE_server"); + let child = Command::new(exe) + .env("CHUSQL_PIPE", &pipe) + .env("CHUSQL_DATA_DIR", data.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let srv = ServerProc(child); + for _ in 0..100 { + if connect(&pipe).is_ok() { break; } + thread::sleep(Duration::from_millis(50)); + } + let mut c = connect(&pipe).unwrap(); + for (id, name) in [(1, "Alice"), (2, "Bob"), (3, "Carol")] { + let req = format!( + r#"{{"method":"insert","table":"persist","row":{{"id":{},"name":"{}"}},"key":{}}}"#, + id, name, id + ); + send(&mut c, &req); + } + drop(srv); // 杀 server + } + + // 等一下，让文件句柄真正释放 + thread::sleep(Duration::from_millis(200)); + + // 第二次：重开 server，从索引查到 + let (_srv, _data) = { + let child = Command::new(env!("CARGO_BIN_EXE_server")) + .env("CHUSQL_PIPE", &pipe) + .env("CHUSQL_DATA_DIR", data.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let srv = ServerProc(child); + for _ in 0..100 { + if connect(&pipe).is_ok() { break; } + thread::sleep(Duration::from_millis(50)); + } + (srv, data) + }; + + let mut c = connect(&pipe).unwrap(); + let r = send(&mut c, r#"{"method":"lookup_by_index","table":"persist","key":2}"#); + assert!(r.contains("Bob"), "persist lookup: {}", r); +} + +/// 配置文件里的管道名和数据目录要真的生效（这里不设 CHUSQL_PIPE / CHUSQL_DATA_DIR）。 +#[test] +fn config_file_is_used() { + let dir = tempfile::tempdir().unwrap(); + let pipe = unique_pipe_name(); + let data_dir = dir.path().join("data"); + let config_path = dir.path().join("chusql-storage.toml"); + + // TOML 里写正斜杠，省得转义 Windows 路径 + let text = format!( + "[server]\npipe_name = \"{}\"\n[storage]\ndata_dir = \"{}\"\n[log]\nlevel = \"debug\"\n", + pipe, + data_dir.display().to_string().replace('\\', "/") + ); + std::fs::write(&config_path, text).unwrap(); + + let child = Command::new(env!("CARGO_BIN_EXE_server")) + .env("CHUSQL_CONFIG", &config_path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to spawn server"); + let _srv = ServerProc(child); + for _ in 0..100 { + if connect(&pipe).is_ok() { + break; + } + thread::sleep(Duration::from_millis(50)); + } + + let mut c = connect(&pipe).unwrap(); + assert_eq!(send(&mut c, r#"{"method":"ping"}"#), r#"{"status":"pong"}"#); + let r = send(&mut c, r#"{"method":"insert","table":"cfg","row":{"id":1}}"#); + assert!(r.contains(r#""status":"ok""#), "insert: {}", r); + + // 数据真的落在配置文件指的那个目录里 + assert!( + data_dir.join("cfg.db").exists(), + "data dir from config file was not used" + ); +} diff --git a/chusql-storage/tests/wal_test.rs b/chusql-storage/tests/wal_test.rs new file mode 100644 index 0000000..aa55a0a --- /dev/null +++ b/chusql-storage/tests/wal_test.rs @@ -0,0 +1,119 @@ +use chusql_storage::protocol::Row; +use chusql_storage::wal::{Wal, WalOp}; +use serde_json::json; + +fn row(pairs: &[(&str, serde_json::Value)]) -> Row { + let mut m = Row::new(); + for (k, v) in pairs { + m.insert((*k).to_string(), v.clone()); + } + m +} + +#[test] +fn append_and_read_insert() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + let op = WalOp::Insert { + table: "users".into(), + row: row(&[("id", json!(1)), ("name", json!("Alice"))]), + key: Some(1), + }; + wal.append(&op).unwrap(); + + let back = wal.read().unwrap().unwrap(); + assert_eq!(back, op); +} + +#[test] +fn append_and_read_replace_all() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + let op = WalOp::ReplaceAll { + table: "users".into(), + rows: vec![ + row(&[("id", json!(1)), ("name", json!("Alice"))]), + row(&[("id", json!(2)), ("name", json!("Bob"))]), + ], + }; + wal.append(&op).unwrap(); + + let back = wal.read().unwrap().unwrap(); + assert_eq!(back, op); +} + +#[test] +fn missing_file_returns_none() { + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + assert!(wal.read().unwrap().is_none()); +} + +#[test] +fn empty_file_returns_none() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wal.log"); + std::fs::write(&path, b"").unwrap(); + let wal = Wal::new(&path); + assert!(wal.read().unwrap().is_none()); +} + +#[test] +fn clear_truncates() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wal.log"); + let wal = Wal::new(&path); + + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(1))]), + key: None, + }) + .unwrap(); + assert!(wal.read().unwrap().is_some()); + + wal.clear().unwrap(); + assert!(wal.read().unwrap().is_none()); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0); +} + +#[test] +fn truncated_tail_is_ignored() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wal.log"); + let wal = Wal::new(&path); + + wal.append(&WalOp::Insert { + table: "users".into(), + row: row(&[("id", json!(1)), ("name", json!("Alice"))]), + key: Some(1), + }) + .unwrap(); + + // 砍掉最后 5 个字节，模拟"写一半断电" + let full = std::fs::read(&path).unwrap(); + std::fs::write(&path, &full[..full.len() - 5]).unwrap(); + + // 读回来应该当作"没有这条" + assert!(wal.read().unwrap().is_none()); +} + +#[test] +fn overwrite_on_second_append_is_visible() { + // 简化版一次只允许一条，append 两次会叠在文件里—— + // 但正常路径下 clear 先跑，所以实际不会叠加。 + // 这个测试只验证 read 能解析最前面那一条。 + let dir = tempfile::tempdir().unwrap(); + let wal = Wal::new(dir.path().join("wal.log")); + + wal.append(&WalOp::Insert { + table: "t".into(), + row: row(&[("id", json!(1))]), + key: None, + }) + .unwrap(); + let first = wal.read().unwrap().unwrap(); + assert!(matches!(first, WalOp::Insert { .. })); +}